commit cb9698951c8c57b6cdb185c73db1dec42f4574ec
author John Koleszar <jkoleszar@google.com> Thu Dec 09 15:59:22 2010 -0500
committer John Koleszar <jkoleszar@google.com> Thu Dec 09 16:05:26 2010 -0500
tree 9bff757c03451678e49d7a97e4124b5486f8b46e
parent c63fc881e109de884e98959ecabcc057fbba2d32

fix uninitialized read in encode breakout

Change I3430820 performed an uninitialized read when
encode_breakout == 0, since the sum and sse wouldn't be set:

   if(x->encode_breakout)
       VARIANCE_INVOKE(..., get16x16var)(..., &sum, &sse);
   if (cpi->active_map_enabled && x->active_ptr[0] == 0) {
       ...
   } else if (sse < x->encode_breakout)

Change-Id: I915eb76d1227b4b6d1137a0dedf2c143860098a2

vp8/encoder/rdopt.c

diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index bc5fd40..9b8b3b3 100644
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -1642,8 +1642,6 @@
     //int intermodecost[MAX_MODES];
 
     MB_PREDICTION_MODE uv_intra_mode;
-    int sse;
-    int sum;
     int uvintra_eob = 0;
     int tteob = 0;
     int force_no_skip = 0;
@@ -2238,36 +2236,44 @@
 
             vp8_set_mbmode_and_mvs(x, this_mode, &mode_mv[this_mode]);
             vp8_build_inter_predictors_mby(&x->e_mbd);
-            if(x->encode_breakout)
-                VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var)(x->src.y_buffer, x->src.y_stride, x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum);
 
             if (cpi->active_map_enabled && x->active_ptr[0] == 0) {
                 x->skip = 1;
             }
-            else if (sse < x->encode_breakout)
+            else if (x->encode_breakout)
             {
-                // Check u and v to make sure skip is ok
-                int sse2 = 0;
+                int sum, sse;
 
-                // add dc check
-                if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2))
+                VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var)
+                    (x->src.y_buffer, x->src.y_stride,
+                     x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum);
+
+                if (sse < x->encode_breakout)
                 {
-                    sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance));
+                    // Check u and v to make sure skip is ok
+                    int sse2 = 0;
 
-                    if (sse2 * 2 < x->encode_breakout)
+                    // add dc check
+                    if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2))
                     {
-                        x->skip = 1;
-                        distortion2 = sse;
-                        rate2 = 500;
-                        disable_skip = 1;
-                        this_rd = RDFUNC(x->rdmult, x->rddiv, rate2,
-                          distortion2, cpi->target_bits_per_mb);
+                        sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance));
 
-                        break;
+                        if (sse2 * 2 < x->encode_breakout)
+                        {
+                            x->skip = 1;
+                            distortion2 = sse;
+                            rate2 = 500;
+                            disable_skip = 1;
+                            this_rd = RDFUNC(x->rdmult, x->rddiv, rate2,
+                              distortion2, cpi->target_bits_per_mb);
+
+                            break;
+                        }
                     }
                 }
             }
 
+
             //intermodecost[mode_index] = vp8_cost_mv_ref(this_mode, mdcounts);   // Experimental debug code
 
             // Add in the Mv/mode cost