commit 215084b31c3e5683c75a558911172d8f34661333
author James Zern <jzern@google.com> Tue Mar 05 12:02:28 2013 -0800
committer James Zern <jzern@google.com> Tue Mar 05 16:18:24 2013 -0800
tree 6746ba56e313de18a81b98b4b7cb1dc4967ec717
parent 7d8fc26c9c8bb99f8c8fb0c860c69630aa6dfec3

vp8: clamp probability values

fixes out of bounds reads on vp8_prob_cost table.
present since:
217591f Added rate-targeted temporal scalability
v0.9.7-p1-71-g217591f

Change-Id: I9194b773098a381f1e3f67bd3307f05df27c24fd

vp8/encoder/bitstream.c

diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c
index e666b6c..8f94171 100644
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -980,6 +980,12 @@
                               int prob_garf
                              )
 {
+    assert(prob_intra >= 0);
+    assert(prob_intra <= 255);
+    assert(prob_last >= 0);
+    assert(prob_last <= 255);
+    assert(prob_garf >= 0);
+    assert(prob_garf <= 255);
     ref_frame_cost[INTRA_FRAME]   = vp8_cost_zero(prob_intra);
     ref_frame_cost[LAST_FRAME]    = vp8_cost_one(prob_intra)
                                     + vp8_cost_zero(prob_last);

vp8/encoder/onyx_if.c

diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 92f9818..555a9e4 100644
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -2816,6 +2816,8 @@
         if (cpi->common.refresh_alt_ref_frame)
         {
             cpi->prob_intra_coded += 40;
+            if (cpi->prob_intra_coded > 255)
+                cpi->prob_intra_coded = 255;
             cpi->prob_last_coded = 200;
             cpi->prob_gf_coded = 1;
         }