Fix signed integer overflow bug STATS_CHANGED BUG=aomedia:3112 This commit fix the signed integer overflow bug when pruning modes based on TPL stats are enabled. The bug appeared when both enable_orip and prune_inter_modes_based_on_tpl are enabled. Change-Id: I720dc5332ad9ad9cecee615f7f5693926d1e7f3d
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 15046eb..5a8b113 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -2672,6 +2672,7 @@ // Prune the mode if cur_inter_cost is greater than threshold times // best_inter_cost const int64_t best_inter_cost = inter_cost_info_from_tpl->best_inter_cost; + if (best_inter_cost == INT64_MAX) return 0; if (cur_inter_cost > ((tpl_inter_mode_prune_mul_factor[prune_level][prune_index] * best_inter_cost) >>