Fix RDO bug when smooth interintra is disabled

If smooth interintra is disabled and no valid wedge interintra mode
is picked, handle_interintra_mode() needs to return -1. Otherwise,
an invalid mbmi will be passed on in RDO.

Change-Id: Idd023a8670354eb2ff3bead2c2f0b7145d2e8c77

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index cb3bdf6..9198133 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -8920,6 +8920,9 @@
         rd = RDCOST(x->rdmult, rmode + tmp_rate_mv + rwedge + rate_sum,
                     dist_sum);
       best_interintra_rd_wedge = rd;
+      if (!cpi->oxcf.enable_smooth_interintra &&
+          best_interintra_rd_wedge == INT64_MAX)
+        return -1;
       if (best_interintra_rd_wedge < best_interintra_rd_nowedge) {
         mbmi->use_wedge_interintra = 1;
         mbmi->mv[0].as_int = tmp_mv.as_int;
@@ -8931,9 +8934,10 @@
         av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, orig_dst, bsize);
       }
     } else {
+      if (!cpi->oxcf.enable_smooth_interintra) return -1;
       mbmi->use_wedge_interintra = 0;
     }
-  }  // if (is_interintra_wedge_used(bsize))
+  }  // if (is_wedge_used)
   if (num_planes > 1) {
     av1_build_inter_predictors_sbuv(cm, xd, mi_row, mi_col, orig_dst, bsize);
   }