Another fix of daala-dist for cb4x4
Daala-dist replaces the luma distortion of sub8x8 partitions with
its own distortion thus requires to split the luma distortion only.
Doing so, there has been a bug that INT_MAX64 value comes
when the sub8x8 parition is skipped. This happened because the existing
code does not initialize the rd_stats_y or tmp_rd_stats_y, i.e. rd_stat struct
for luma only in several places.
Change-Id: If229b53bb7a6cff0b8751138a32b1dcf02665624
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index a0eed78..cf98b35 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -3551,6 +3551,11 @@
&this_rdc, best_rdc.rdcost - sum_rdc.rdcost, pc_tree->split[idx]);
#endif // CONFIG_SUPERTX
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if (bsize == BLOCK_8X8 && this_rdc.rate != INT_MAX) {
+ assert(this_rdc.dist_y < INT64_MAX);
+ }
+#endif
if (this_rdc.rate == INT_MAX) {
sum_rdc.rdcost = INT64_MAX;
#if CONFIG_SUPERTX
@@ -3565,7 +3570,10 @@
sum_rate_nocoef += this_rate_nocoef;
#endif // CONFIG_SUPERTX
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
- sum_rdc.dist_y += this_rdc.dist_y;
+ if (bsize == BLOCK_8X8) {
+ assert(this_rdc.dist_y < INT64_MAX);
+ sum_rdc.dist_y += this_rdc.dist_y;
+ }
#endif
}
}
@@ -3581,6 +3589,7 @@
src_stride, x->decoded_8x8, 8, 8, 8, 1,
use_activity_masking, x->qindex)
<< 4;
+ assert(sum_rdc.dist_y < INT64_MAX);
sum_rdc.dist = sum_rdc.dist - sum_rdc.dist_y + daala_dist;
sum_rdc.rdcost =
RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
@@ -4068,6 +4077,12 @@
// checks occur in some sub function and thus are used...
(void)best_rd;
*rd_cost = best_rdc;
+
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if (bsize <= BLOCK_8X8 && rd_cost->rate != INT_MAX) {
+ assert(rd_cost->dist_y < INT64_MAX);
+ }
+#endif // CONFIG_DAALA_DIST && CONFIG_CB4X4
#if CONFIG_SUPERTX
*rate_nocoef = best_rate_nocoef;
#endif // CONFIG_SUPERTX
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 720a3df..953ce90 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -9498,6 +9498,19 @@
{
RD_STATS rd_stats, rd_stats_y, rd_stats_uv;
av1_init_rd_stats(&rd_stats);
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ // While av1 master uses rd_stats_y.rate through out the codebase,
+ // which is set when handle_inter_moden is called, the daala-dist code
+ // in rd_pick_partition() for cb4x4 and sub8x8 blocks need to know
+ // .dist_y which comes from rd_stats_y.dist and rd_stats_y.sse.
+ // The problem is rd_stats_y.dist and rd_stats_y.sse are sometimes not
+ // initialized when rd_stats.skip = 1,
+ // then instead rd_stats.dist and rd_stats.sse have the
+ // combined luma and chroma dist and sse.
+ // This can be seen inside motion_mode_rd(), which is called by
+ // handle_inter_mode().
+ if (bsize < BLOCK_8X8) av1_init_rd_stats(&rd_stats_y);
+#endif
rd_stats.rate = rate2;
// Point to variables that are maintained between loop iterations
@@ -9517,7 +9530,14 @@
rate_y = rd_stats_y.rate;
rate_uv = rd_stats_uv.rate;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
- if (bsize < BLOCK_8X8) distortion2_y = rd_stats_y.dist;
+ if (bsize < BLOCK_8X8) {
+ if (rd_stats_y.rate != INT_MAX) {
+ assert(rd_stats_y.sse < INT64_MAX);
+ assert(rd_stats_y.dist < INT64_MAX);
+ }
+ total_sse_y = rd_stats_y.sse;
+ distortion2_y = rd_stats_y.dist;
+ }
#endif
}
@@ -9585,6 +9605,7 @@
RD_STATS tmp_rd_stats, tmp_rd_stats_y, tmp_rd_stats_uv;
av1_invalid_rd_stats(&tmp_rd_stats);
+
x->skip = 0;
mbmi->ref_mv_idx = 1 + ref_idx;
@@ -9655,7 +9676,15 @@
frame_mv[NEARMV][ref_frame] = cur_mv;
av1_init_rd_stats(&tmp_rd_stats);
-
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ // With the same reason as 'rd_stats_y' passed to above
+ // handle_inter_mode(), tmp_rd_stats_y.dist and
+ // tmp_rd_stats_y.sse are sometimes not initialized, esp. when
+ // tmp_rd_stats.skip = 1 and tmp_rd_stats.dist and .sse
+ // represent combined luma and chroma .dist and .sse,
+ // we should initialized tmp_rd_stats_y.
+ if (bsize < BLOCK_8X8) av1_init_rd_stats(&tmp_rd_stats_y);
+#endif
// Point to variables that are not maintained between iterations
args.single_newmv = dummy_single_newmv;
#if CONFIG_EXT_INTER
@@ -9730,6 +9759,10 @@
backup_skip = x->skip;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
if (bsize < BLOCK_8X8) {
+ if (tmp_rd_stats_y.rate != INT_MAX) {
+ assert(tmp_rd_stats_y.sse < INT64_MAX);
+ assert(tmp_rd_stats_y.dist < INT64_MAX);
+ }
total_sse_y = tmp_rd_stats_y.sse;
distortion2_y = tmp_rd_stats_y.dist;
}
@@ -9808,7 +9841,10 @@
rate_y = 0;
rate_uv = 0;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
- if (bsize < BLOCK_8X8) distortion2_y = total_sse_y;
+ if (bsize < BLOCK_8X8) {
+ assert(total_sse_y < INT64_MAX);
+ distortion2_y = total_sse_y;
+ }
#endif
}
} else {
@@ -9829,6 +9865,12 @@
#endif // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
}
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if ((bsize < BLOCK_8X8) && (rate2 != INT_MAX)) {
+ assert(distortion2_y < INT64_MAX);
+ }
+#endif
+
if (ref_frame == INTRA_FRAME) {
// Keep record of best intra rd
if (this_rd < best_intra_rd) {
@@ -9900,7 +9942,10 @@
this_skip2 || skippable);
best_rate_uv = rate_uv;
#if CONFIG_DAALA_DIST && CONFIG_CB4X4
- if (bsize < BLOCK_8X8) rd_cost->dist_y = distortion2_y;
+ if (bsize < BLOCK_8X8) {
+ assert(distortion2_y < INT64_MAX);
+ rd_cost->dist_y = distortion2_y;
+ }
#endif
#if CONFIG_VAR_TX
for (i = 0; i < MAX_MB_PLANE; ++i)
@@ -9909,7 +9954,11 @@
#endif // CONFIG_VAR_TX
}
}
-
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if ((bsize < BLOCK_8X8) && (rd_cost->rate != INT_MAX)) {
+ assert(rd_cost->dist_y < INT64_MAX);
+ }
+#endif
/* keep record of best compound/single-only prediction */
if (!disable_skip && ref_frame != INTRA_FRAME) {
int64_t single_rd, hybrid_rd, single_rate, hybrid_rate;
@@ -10029,15 +10078,25 @@
rd_cost->rate +=
(rd_stats_y.rate + rd_stats_uv.rate - best_rate_y - best_rate_uv);
rd_cost->dist = rd_stats_y.dist + rd_stats_uv.dist;
-#if CONFIG_DAALA_DIST && CONFIG_CB4X4
- if (bsize < BLOCK_8X8) rd_cost->dist_y = rd_stats_y.dist;
-#endif
rd_cost->rdcost =
RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist);
best_skip2 = skip_blk;
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if (bsize < BLOCK_8X8) {
+ assert(rd_cost->rate != INT_MAX);
+ assert(rd_cost->dist_y < INT64_MAX);
+ rd_cost->dist_y = rd_stats_y.dist;
+ }
+#endif
}
}
+#if CONFIG_DAALA_DIST && CONFIG_CB4X4
+ if ((bsize < BLOCK_8X8) && (rd_cost->rate != INT_MAX)) {
+ assert(rd_cost->dist_y < INT64_MAX);
+ }
+#endif
+
#if CONFIG_PALETTE
// Only try palette mode when the best mode so far is an intra mode.
if (try_palette && !is_inter_mode(best_mbmode.mode)) {
