commit 06a55bcbdb726f81f2f13aa0a51ee4181568ba19
author bohanli <bohanli@google.com> Thu Jan 16 14:26:09 2020 -0800
committer Bohan Li <bohanli@google.com> Fri Jan 17 19:36:13 2020 +0000
tree 39178d1a5e0557279e0881615365d69d7c89affa
parent b4410d8acd6c04c6eced35459eaf9c41b628ca51

Fix a potential divided by zero bug.

Supposedly estimated rd cost should be always greater than zero.
This is a temporary fix to ensure it.

Will look further to check how divided by zero happened.

Change-Id: I7f9e5b4e41f0a0cb31a52bdba9f3ca54a7b44bf6

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 58979c3..538c67f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2422,6 +2422,7 @@
 
     txk_map[num_cand] = tx_type;
     rds[num_cand] = RDCOST(x->rdmult, rate_cost, dist);
+    if (rds[num_cand] == 0) rds[num_cand] = 1;
     num_cand++;
   }
 
@@ -2555,6 +2556,7 @@
     } else {
       txk_map[num_cand] = tx_type;
       rds[num_cand] = rds_v[i_v] + rds_h[i_h];
+      if (rds[num_cand] == 0) rds[num_cand] = 1;
       num_cand++;
     }
   }