commit 01b01c0d12d19b4e66a54f6e48fa8aaa545a629e
author Yue Chen <yuec@google.com> Fri Nov 17 15:52:46 2017 -0800
committer Yue Chen <yuec@google.com> Sun Nov 19 00:07:22 2017 +0000
tree 48cb1873cf6e35aa00a039b511da1078a641d62b
parent b433f4c40bbcdbaa88258445116a9850b49d5dc1

Make the first cdf prob < CDF_PROB_TOP to avoid infinite loop

The probability training tool generates all CDF_PROB_TOP table if
the counts are all zero. Although it means such cases are never
used in actual encoding, in rd cost pre-calculation, we still
sometimes run into such cases and the token cost calcualtion
function cannot handle all-CDF_PROB_TOP cdfs (Will trigger assertion
when debug mode is on).

Change-Id: I259620e210fea0f6e1f7848724fa006fa06da226

tools/aom_entropy_optimizer.c

diff --git a/tools/aom_entropy_optimizer.c b/tools/aom_entropy_optimizer.c
index 8acdc0d..3108a0d 100644
--- a/tools/aom_entropy_optimizer.c
+++ b/tools/aom_entropy_optimizer.c
@@ -212,6 +212,7 @@
     else
       cdf[i] = (csum[i] * CDF_PROB_TOP + round_shift) / sum;
   }
+  if (sum <= 0) cdf[0] = CDF_PROB_TOP - 1;
   return 0;
 }