commit 7b6376984075dbffb77903b813997d571a4df356
author Fyodor Kyslov <kyslov@google.com> Thu Oct 07 10:26:48 2021 -0700
committer Wan-Teh Chang <wtc@google.com> Fri Oct 08 23:15:17 2021 +0000
tree 09a3523f3e52ab1b68ed79b25750306137f826bc
parent bfc5538fa2c3703a530169e3cccef23d73c804b9

AV1 RT: Fix uninitialized variable use in estimate_intra

Accessing MVs without checking any inter mode leads to MSAN warnings

BUG=b/200884145

Change-Id: I1ca8ab90da567cd4a33ffa4c3c4caa185d67d33d
(cherry picked from commit 552a0398f47d558b315b4ee0b6b2ec31606a8706)

av1/encoder/nonrd_pickmode.c

diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 3f17713..a3f1216 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1986,9 +1986,10 @@
   if (x->source_variance < spatial_var_thresh) {
     // If the best inter mode is large motion or non-LAST ref reduce intra cost
     // penalty, so intra mode is more likely tested.
-    if (best_pickmode->best_ref_frame != LAST_FRAME ||
-        abs(mi->mv[0].as_mv.row) >= motion_thresh ||
-        abs(mi->mv[0].as_mv.col) >= motion_thresh) {
+    if (best_rdc->rdcost != INT64_MAX &&
+        (best_pickmode->best_ref_frame != LAST_FRAME ||
+         abs(mi->mv[0].as_mv.row) >= motion_thresh ||
+         abs(mi->mv[0].as_mv.col) >= motion_thresh)) {
       intra_cost_penalty = intra_cost_penalty >> 2;
       inter_mode_thresh = RDCOST(x->rdmult, intra_cost_penalty, 0);
       do_early_exit_rdthresh = 0;