commit 451e0f22c00cad5820a0b9b5e62ca568d5947720
author Urvang Joshi <urvang@google.com> Tue Jan 31 11:18:31 2017 -0800
committer Urvang Joshi <urvang@google.com> Tue Jan 31 22:05:22 2017 +0000
tree 47f0078fb5c06a3fa55823fb7dded4657132c56d
parent 4415b6ccd9e4693dcd8772c10fa449641e53db95

Bugfix: Fix an assert being triggered because of palette.

rd_pick_palette_intra_sby() was being passed a dummy 'best_mbmi' in
one case, and later that dummy value could be assigned to 'mbmi'
causing invalid values in 'mbmi'.

This bug was introduced by:
de0c70a2: Refactor rd_pick_intra_sby_mode()

See the referenced bug for a repro case.

BUG=aomedia:337

Change-Id: If6b0aa329f4e4902cd4de8a5e09bc5d9196492a1

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 38efddf..34780d0 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -10687,22 +10687,24 @@
 #if CONFIG_SUPERTX
     int best_rate_nocoef;
 #endif
-    int64_t distortion2 = 0, dummy_rd = best_rd, this_rd, model_rd = INT64_MAX;
+    int64_t distortion2 = 0, best_rd_palette = best_rd, this_rd,
+            best_model_rd_palette = INT64_MAX;
     int skippable = 0, rate_overhead_palette = 0;
     RD_STATS rd_stats_y;
     TX_SIZE uv_tx;
     uint8_t *const best_palette_color_map =
         x->palette_buffer->best_palette_color_map;
     uint8_t *const color_map = xd->plane[0].color_index_map;
-    MB_MODE_INFO mbmi_dummy;
+    MB_MODE_INFO best_mbmi_palette = best_mbmode;
 
     mbmi->mode = DC_PRED;
     mbmi->uv_mode = DC_PRED;
     mbmi->ref_frame[0] = INTRA_FRAME;
     mbmi->ref_frame[1] = NONE_FRAME;
     rate_overhead_palette = rd_pick_palette_intra_sby(
-        cpi, x, bsize, palette_ctx, intra_mode_cost[DC_PRED], &mbmi_dummy,
-        best_palette_color_map, &dummy_rd, &model_rd, NULL, NULL, NULL, NULL);
+        cpi, x, bsize, palette_ctx, intra_mode_cost[DC_PRED],
+        &best_mbmi_palette, best_palette_color_map, &best_rd_palette,
+        &best_model_rd_palette, NULL, NULL, NULL, NULL);
     if (pmi->palette_size[0] == 0) goto PALETTE_EXIT;
     memcpy(color_map, best_palette_color_map,
            rows * cols * sizeof(best_palette_color_map[0]));