commit 2becffaef5aad04b6f73500774b4151ad84ab83a
author Yaowu Xu <yaowu@google.com> Tue Mar 15 12:36:21 2016 -0700
committer Yaowu Xu <yaowu@google.com> Tue Mar 15 15:00:53 2016 -0700
tree b7290502efcb2f670b249b1a372e6ac391b60488
parent dcbbb81605cca975e47e23fc2fdb951c8ac2d037

Properly set the estimate of rate_nocoef

This commit fixes the computation of rate_nocoef for situation when
rate_y is uninitialized at INT_MAX for  x->skip is true.

Change-Id: If3dde4e4ee16667f4408067d3bb3084f916272f1

vp10/encoder/encodeframe.c

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index c5a68a9..c66be39 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -1654,6 +1654,7 @@
                                    totalrate_nocoef,
 #endif  // CONFIG_SUPERTX
                                    bsize, ctx, best_rd);
+        assert(*totalrate_nocoef >= 0);
       }
     } else {
       vp10_rd_pick_inter_mode_sub8x8(cpi, tile_data, x, mi_row, mi_col, rd_cost,
@@ -1661,6 +1662,7 @@
                                      totalrate_nocoef,
 #endif  // CONFIG_SUPERTX
                                      bsize, ctx, best_rd);
+      assert(*totalrate_nocoef >= 0);
     }
   }
 

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 6543b33..830761d 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -8333,11 +8333,12 @@
 
         rd_cost->rate = rate2;
 #if CONFIG_SUPERTX
-        *returnrate_nocoef = rate2 - rate_y - rate_uv;
-        if (!disable_skip) {
-          *returnrate_nocoef -= vp10_cost_bit(vp10_get_skip_prob(cm, xd),
-                                              skippable || this_skip2);
-        }
+        if (x->skip && rate_y == INT_MAX)
+          *returnrate_nocoef = rate2;
+        else
+          *returnrate_nocoef = rate2 - rate_y - rate_uv;
+        *returnrate_nocoef -= vp10_cost_bit(vp10_get_skip_prob(cm, xd),
+            disable_skip || skippable || this_skip2);
         *returnrate_nocoef -= vp10_cost_bit(vp10_get_intra_inter_prob(cm, xd),
                                             mbmi->ref_frame[0] != INTRA_FRAME);
 #if CONFIG_OBMC