Avoid changing motion_mode after rd search

Check if the global motion undergoes non-translation mode. If that
is the case, do not force a newmv mode into globalmv mode.

This will resolve a mismatch bug of txk_sel.

Change-Id: I589f1619427f02045f8cf49bdea05f348f076236

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 0825675..0e3da84 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -2496,6 +2496,7 @@
   if (plane == 0)
     mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = best_tx_type;
   x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx;
+  x->plane[plane].eobs[block] = best_eob;
 
   if (!is_inter_block(mbmi)) {
     // intra mode needs decoded result such that the next transform block

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index c7713d8..d0fbe6b 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -10686,6 +10686,14 @@
                                                   )
                                  .as_int
                            : 0;
+
+    // Check if the global motion mode is non-translational.
+    int is_nontran_gm = cm->global_motion[refs[0]].wmtype <= TRANSLATION;
+    if (comp_pred_mode)
+      is_nontran_gm &= cm->global_motion[refs[1]].wmtype <= TRANSLATION;
+    if (AOMMIN(block_size_wide[bsize], block_size_high[bsize]) < 8)
+      is_nontran_gm = 1;
+
     if (!comp_pred_mode) {
       int ref_set = (mbmi_ext->ref_mv_count[rf_type] >= 2)
                         ? AOMMIN(2, mbmi_ext->ref_mv_count[rf_type] - 2)
@@ -10701,7 +10709,7 @@
 
       if (frame_mv[NEARESTMV][refs[0]].as_int == best_mbmode.mv[0].as_int)
         best_mbmode.mode = NEARESTMV;
-      else if (best_mbmode.mv[0].as_int == zeromv[0].as_int)
+      else if (best_mbmode.mv[0].as_int == zeromv[0].as_int && is_nontran_gm)
         best_mbmode.mode = GLOBALMV;
     } else {
       int_mv nearestmv[2];
@@ -10744,7 +10752,7 @@
 
         if (best_mbmode.mode == NEW_NEWMV &&
             best_mbmode.mv[0].as_int == zeromv[0].as_int &&
-            best_mbmode.mv[1].as_int == zeromv[1].as_int)
+            best_mbmode.mv[1].as_int == zeromv[1].as_int && is_nontran_gm)
           best_mbmode.mode = GLOBAL_GLOBALMV;
       }
     }
@@ -10815,13 +10823,11 @@
     // Correct the motion mode for GLOBALMV
     const MOTION_MODE last_motion_mode_allowed =
         motion_mode_allowed(xd->global_motion, xd, xd->mi[0]);
-    if (mbmi->motion_mode > last_motion_mode_allowed)
-      mbmi->motion_mode = last_motion_mode_allowed;
-
-    // Correct the interpolation filter for GLOBALMV
+    if (mbmi->motion_mode > last_motion_mode_allowed) assert(0);
     if (is_nontrans_global_motion(xd)) {
-      mbmi->interp_filters = av1_broadcast_interp_filter(
-          av1_unswitchable_filter(cm->interp_filter));
+      assert(mbmi->interp_filters ==
+             av1_broadcast_interp_filter(
+                 av1_unswitchable_filter(cm->interp_filter)));
     }
   }