Another fix of daala-dist for cb4x4
Daala-dist replaces the luma distortion of sub8x8 partitions with
its own distortion thus requires to split the luma distortion only.
Doing so, there has been a bug that INT_MAX64 value comes
when the sub8x8 parition is skipped. This happened because the existing
code does not initialize the rd_stats_y or tmp_rd_stats_y, i.e. rd_stat struct
for luma only in several places.
Change-Id: If229b53bb7a6cff0b8751138a32b1dcf02665624
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index a0eed78..cf98b35 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -3551,6 +3551,11 @@
&this_rdc, best_rdc.rdcost - sum_rdc.rdcost, pc_tree->split[idx]);
#endif // CONFIG_SUPERTX
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if (bsize == BLOCK_8X8 && this_rdc.rate != INT_MAX) {
+ assert(this_rdc.dist_y < INT64_MAX);
+ }
+#endif
if (this_rdc.rate == INT_MAX) {
sum_rdc.rdcost = INT64_MAX;
#if CONFIG_SUPERTX
@@ -3565,7 +3570,10 @@
sum_rate_nocoef += this_rate_nocoef;
#endif // CONFIG_SUPERTX
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
- sum_rdc.dist_y += this_rdc.dist_y;
+ if (bsize == BLOCK_8X8) {
+ assert(this_rdc.dist_y < INT64_MAX);
+ sum_rdc.dist_y += this_rdc.dist_y;
+ }
#endif
}
}
@@ -3581,6 +3589,7 @@
src_stride, x->decoded_8x8, 8, 8, 8, 1,
use_activity_masking, x->qindex)
<< 4;
+ assert(sum_rdc.dist_y < INT64_MAX);
sum_rdc.dist = sum_rdc.dist - sum_rdc.dist_y + daala_dist;
sum_rdc.rdcost =
RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
@@ -4068,6 +4077,12 @@
// checks occur in some sub function and thus are used...
(void)best_rd;
*rd_cost = best_rdc;
+
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if (bsize <= BLOCK_8X8 && rd_cost->rate != INT_MAX) {
+ assert(rd_cost->dist_y < INT64_MAX);
+ }
+#endif // CONFIG_DAALA_DIST && CONFIG_CB4X4
#if CONFIG_SUPERTX
*rate_nocoef = best_rate_nocoef;
#endif // CONFIG_SUPERTX
