commit 552a0398f47d558b315b4ee0b6b2ec31606a8706
author Fyodor Kyslov <kyslov@google.com> Thu Oct 07 10:26:48 2021 -0700
committer Fyodor Kyslov <kyslov@google.com> Thu Oct 07 10:49:57 2021 -0700
tree 8e336f9962a9828362eb6a48306c429946a72840
parent 10274b5578ae79c999e07482131c30e7d8d4c6af

AV1 RT: Fix uninitialized variable use in estimate_intra

Accessing MVs without checking any inter mode leads to MSAN warnings

BUG=b/200884145

Change-Id: I1ca8ab90da567cd4a33ffa4c3c4caa185d67d33d

av1/encoder/nonrd_pickmode.c

diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 4b12353..5ee5761 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -2028,9 +2028,10 @@
   if (x->source_variance < spatial_var_thresh) {
     // If the best inter mode is large motion or non-LAST ref reduce intra cost
     // penalty, so intra mode is more likely tested.
-    if (best_pickmode->best_ref_frame != LAST_FRAME ||
-        abs(mi->mv[0].as_mv.row) >= motion_thresh ||
-        abs(mi->mv[0].as_mv.col) >= motion_thresh) {
+    if (best_rdc->rdcost != INT64_MAX &&
+        (best_pickmode->best_ref_frame != LAST_FRAME ||
+         abs(mi->mv[0].as_mv.row) >= motion_thresh ||
+         abs(mi->mv[0].as_mv.col) >= motion_thresh)) {
       intra_cost_penalty = intra_cost_penalty >> 2;
       inter_mode_thresh = RDCOST(x->rdmult, intra_cost_penalty, 0);
       do_early_exit_rdthresh = 0;