commit a3af64597a023f1ddb40fe335e7f2505b81cbd85
author Debargha Mukherjee <debargha@google.com> Wed Nov 14 21:09:32 2018 -0800
committer Debargha Mukherjee <debargha@google.com> Thu Nov 15 17:52:24 2018 +0000
tree 702b025d275d0818d0515f00fa1050bcfcea7ea8
parent ebf96f4eb8a5a3fd4c070fed912faf476a93e4d5

Do not skip mode if no valid mode has been found

Sometimes in fast speed settings, we may not find a valid
mode, resulting in the encoder going into an infinite loop.
This patch adds an exit to prevent this hanging behavior.

BUG=aomedia:2243

Change-Id: I85afe2faa3492e5a9cc034d80df9e3837a08df6c

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 96b8763..f9e8284 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -10970,6 +10970,11 @@
   const MV_REFERENCE_FRAME *ref_frame = av1_mode_order[mode_index].ref_frame;
   const PREDICTION_MODE this_mode = av1_mode_order[mode_index].mode;
   int skip_motion_mode = 0;
+  // If no valid mode has been found so far in PARTITION_NONE when finding a
+  // valid partition is required, do not skip mode.
+  if (search_state->best_rd == INT64_MAX && mbmi->partition == PARTITION_NONE &&
+      x->must_find_valid_partition)
+    return 0;
   if (mbmi->partition != PARTITION_NONE && mbmi->partition != PARTITION_SPLIT) {
     const int ref_type = av1_ref_frame_type(ref_frame);
     int skip_ref = ctx->skip_ref_frame_mask & (1 << ref_type);