commit b862c108ddf071a541979a6dbd210fcbc9db1df1
author Paul Wilkins <paulwilkins@google.com> Thu Feb 24 15:49:41 2011 +0000
committer Paul Wilkins <paulwilkins@google.com> Thu Feb 24 15:49:41 2011 +0000
tree aeaa50d8e97e9c6a6148abcd5bd074dc03f85c90
parent aee120afb98872a49486d28b8315a1feaa9fffe2

Overflow of frame error accumulators.

This fixes an overflow problem in the frame error accumulators.

The overflow condition is extreme but did trigger when Frank B.
coded some high motion interlaced HD content.

The observed effect was a catastrophic  breakdown of the rate
control leading to massive undershoot and poor bit allocation.

All the error values should really be unsigned but I will look at this
separately.

Change-Id: I9745f5c5ca2783620426b66b568b2088b579151f

vp8/encoder/firstpass.c

diff --git a/vp8/encoder/firstpass.c b/vp8/encoder/firstpass.c
index 9f0a972..964e68a 100644
--- a/vp8/encoder/firstpass.c
+++ b/vp8/encoder/firstpass.c
@@ -560,8 +560,8 @@
     YV12_BUFFER_CONFIG *gld_yv12 = &cm->yv12_fb[cm->gld_fb_idx];
     int recon_y_stride = lst_yv12->y_stride;
     int recon_uv_stride = lst_yv12->uv_stride;
-    int intra_error = 0;
-    int coded_error = 0;
+    long long intra_error = 0;
+    long long coded_error = 0;
 
     int sum_mvr = 0, sum_mvc = 0;
     int sum_mvr_abs = 0, sum_mvc_abs = 0;
@@ -648,7 +648,7 @@
             this_error += intrapenalty;
 
             // Cumulative intra error total
-            intra_error += this_error;
+            intra_error += (long long)this_error;
 
             // Indicate default assumption of intra in the motion map
             *fp_motion_map_ptr = 0;
@@ -800,7 +800,7 @@
                 }
             }
 
-            coded_error += this_error;
+            coded_error += (long long)this_error;
 
             // adjust to the next column of macroblocks
             x->src.y_buffer += 16;