commit b3ad81288fa8ac2842884e5b97b59236f27721ff
author Alex Converse <aconverse@google.com> Wed Feb 10 14:12:26 2016 -0800
committer Alex Converse <aconverse@google.com> Thu Feb 11 09:54:24 2016 -0800
tree af695cc1a04a13f331a766bd96712ee03fc78614
parent 00380700fb2ed7f209035bc92dfd30a4ee58b033

Port switch to 9-bit rate cost to vp10.

Brings the following commits to vp10:
269428e Tie the bit cost scale to a define.
d13385c Switch to 9-bit rate cost constants built on a 256 probability denominator.
ad43a73 Fix a signed overflow in vp9 motion cost.
1c9b091 Fix some interger overflow errors
fac947d Restore previous motion search bit-error scale.

Change-Id: I598ba7ee7efcde18439c31dfa96b86cbf297a580

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 9e29ce6..99658e8 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -411,7 +411,7 @@
       int quantizer = (pd->dequant[1] >> dequant_shift);
 
       if (quantizer < 120)
-        rate = (square_error * (280 - quantizer)) >> 8;
+        rate = (square_error * (280 - quantizer)) >> (16 - VP9_PROB_COST_SHIFT);
       else
         rate = 0;
       dist = (square_error * quantizer) >> 8;
@@ -523,9 +523,9 @@
 #endif
   int c, cost;
 #if CONFIG_VP9_HIGHBITDEPTH
-  const int16_t *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
+  const int *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
 #else
-  const int16_t *cat6_high_cost = vp10_get_high_cost_table(8);
+  const int *cat6_high_cost = vp10_get_high_cost_table(8);
 #endif
 
 #if !CONFIG_VAR_TX && !CONFIG_SUPERTX