commit 1742b043e2269dc1927afe428fbf5a46493e741e
author Angie Chiang <angiebird@google.com> Wed Aug 29 15:00:11 2018 -0700
committer Angie Chiang <angiebird@google.com> Fri Aug 31 07:44:30 2018 +0000
tree 76354caec86b2cd58a65208ae67baa4bfb4aa512
parent 2c6fc5e1bf46810061d9e7a4c984d3e2360c2850

Fix overflow issue when updating best_rate_y

When do_tx_search == 0, handle_inter_mode() won't provide the
correct rate_y and rate_uv because txfm_search is replaced by
rd_estimation. Therefore, we should avoid updating best_rate_y
and best_rate_uv in the inter mode search for loop.
Theses two values will be updated later on when txfm_search is
actually called.
BUG=aomedia:2114

Change-Id: Ibbb02996e01e7c3fb15767e413d9c5e9389fc52b

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 7353f4a..763875e 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -11365,10 +11365,24 @@
         search_state.best_mbmode = *mbmi;
         search_state.best_skip2 = this_skip2;
         search_state.best_mode_skippable = skippable;
+#if CONFIG_COLLECT_INTER_MODE_RD_STATS
+        if (do_tx_search) {
+          // When do_tx_search == 0, handle_inter_mode won't provide correct
+          // rate_y and rate_uv because txfm_search process is replaced by
+          // rd estimation.
+          // Therfore, we should avoid updating best_rate_y and best_rate_uv
+          // here. These two values will be updated when txfm_search is called
+          search_state.best_rate_y =
+              rate_y +
+              x->skip_cost[av1_get_skip_context(xd)][this_skip2 || skippable];
+          search_state.best_rate_uv = rate_uv;
+        }
+#else   // CONFIG_COLLECT_INTER_MODE_RD_STATS
         search_state.best_rate_y =
             rate_y +
             x->skip_cost[av1_get_skip_context(xd)][this_skip2 || skippable];
         search_state.best_rate_uv = rate_uv;
+#endif  // CONFIG_COLLECT_INTER_MODE_RD_STATS
         memcpy(ctx->blk_skip, x->blk_skip,
                sizeof(x->blk_skip[0]) * ctx->num_4x4_blk);
       }