commit da3d94fe2c0a993b42c2146ddf6e669fa6be7e0a
author Alex Converse <aconverse@google.com> Wed Mar 15 14:54:29 2017 -0700
committer Alex Converse <aconverse@google.com> Thu Mar 16 16:36:19 2017 +0000
tree 665c0a61f3c602c49bbf4499529b4e3989e3c5c3
parent e063e2dd3dadeb4812189526ab76f4f718ae7e4c

Account for elided extrabits during rate cost calculation.

Fixes some rd-debug mismatches coding cat6 tokens with tx size < 32x32.
For these tokens the high extrabits are elided during tokenization and
detokenization, but the rd cost was computed with the old tables from
VP9 where these high extrabits are always coded.

Change-Id: I4a9a6ea822ff821e1932c351d43a57bdb4d6d466

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 0aaaf68..cd126e7 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1120,9 +1120,9 @@
 #endif  // CONFIG_NEW_TOKENSET
 
 #if CONFIG_AOM_HIGHBITDEPTH
-  const int *cat6_high_cost = av1_get_high_cost_table(xd->bd);
+  const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, xd->bd);
 #else
-  const int *cat6_high_cost = av1_get_high_cost_table(8);
+  const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, 8);
 #endif  // CONFIG_AOM_HIGHBITDEPTH
 
 #if !CONFIG_VAR_TX && !CONFIG_SUPERTX
@@ -1145,7 +1145,7 @@
       // dc token
       int v = qcoeff[0];
       int16_t prev_t;
-      cost = av1_get_token_cost(v, &prev_t, cat6_high_cost);
+      cost = av1_get_token_cost(v, &prev_t, cat6_bits);
 #if CONFIG_NEW_TOKENSET
       cost += (*token_costs)[!prev_t][pt][prev_t];
 #else
@@ -1161,7 +1161,7 @@
         int16_t t;
 
         v = qcoeff[rc];
-        cost += av1_get_token_cost(v, &t, cat6_high_cost);
+        cost += av1_get_token_cost(v, &t, cat6_bits);
 #if CONFIG_NEW_TOKENSET
         cost += (*token_costs)[!t][!prev_t][t];
 #else
@@ -1187,7 +1187,7 @@
 #if !CONFIG_NEW_TOKENSET
       unsigned int(*tok_cost_ptr)[COEFF_CONTEXTS][ENTROPY_TOKENS];
 #endif
-      cost = av1_get_token_cost(v, &tok, cat6_high_cost);
+      cost = av1_get_token_cost(v, &tok, cat6_bits);
 #if CONFIG_NEW_TOKENSET
       cost += (*token_costs)[!tok][pt][tok];
 #else
@@ -1206,7 +1206,7 @@
         const int rc = scan[c];
 
         v = qcoeff[rc];
-        cost += av1_get_token_cost(v, &tok, cat6_high_cost);
+        cost += av1_get_token_cost(v, &tok, cat6_bits);
         pt = get_coef_context(nb, token_cache, c);
 #if CONFIG_NEW_TOKENSET
         cost += (*token_costs)[!tok][pt][tok];