Bugfix of signed integer overflow

BUG=aomedia:2009

Change-Id: Ic05f772d4c9c469cf30afc5ff67de4438b0ae112

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 8181f3b..ae391c9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -8817,7 +8817,7 @@
       restore_dst_buf(xd, orig_dst, num_planes);
       continue;
     } else if (cpi->sf.model_based_post_interp_filter_breakout &&
-               ref_best_rd != INT64_MAX && (rd > ref_best_rd * 6)) {
+               ref_best_rd != INT64_MAX && (rd / 6 > ref_best_rd)) {
       early_terminate = INT64_MAX;
       restore_dst_buf(xd, orig_dst, num_planes);
       if ((rd >> 4) > ref_best_rd) break;