Fix RDO bug when smooth interintra is disabled
If smooth interintra is disabled and no valid wedge interintra mode
is picked, handle_interintra_mode() needs to return -1. Otherwise,
an invalid mbmi will be passed on in RDO.
Change-Id: Idd023a8670354eb2ff3bead2c2f0b7145d2e8c77
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index cb3bdf6..9198133 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -8920,6 +8920,9 @@
rd = RDCOST(x->rdmult, rmode + tmp_rate_mv + rwedge + rate_sum,
dist_sum);
best_interintra_rd_wedge = rd;
+ if (!cpi->oxcf.enable_smooth_interintra &&
+ best_interintra_rd_wedge == INT64_MAX)
+ return -1;
if (best_interintra_rd_wedge < best_interintra_rd_nowedge) {
mbmi->use_wedge_interintra = 1;
mbmi->mv[0].as_int = tmp_mv.as_int;
@@ -8931,9 +8934,10 @@
av1_build_inter_predictors_sby(cm, xd, mi_row, mi_col, orig_dst, bsize);
}
} else {
+ if (!cpi->oxcf.enable_smooth_interintra) return -1;
mbmi->use_wedge_interintra = 0;
}
- } // if (is_interintra_wedge_used(bsize))
+ } // if (is_wedge_used)
if (num_planes > 1) {
av1_build_inter_predictors_sbuv(cm, xd, mi_row, mi_col, orig_dst, bsize);
}