Fix skip_txfm issue in rdopt code Fixed an encoder crash. Set skip_txfm to 0 for cases that skip_txfm isn't calculated. Put memcpy of skip_txfm at right place. Change-Id: Ib3b6afc1b251a85b2a853c8138fb3393f48cfef6
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 20a4b70..0498a10 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c
@@ -211,6 +211,7 @@ x->bsse[(i << 2) + block_idx] = sse; sum_sse += sse; + x->skip_txfm[(i << 2) + block_idx] = 0; if (!x->select_tx_size) { // Check if all ac coefficients can be quantized to zero. if (var < p->quant_thred[1] >> shift) { @@ -219,8 +220,6 @@ // Check if dc coefficient can be quantized to zero. if (sse - var < p->quant_thred[0] >> shift) x->skip_txfm[(i << 2) + block_idx] = 1; - } else { - x->skip_txfm[(i << 2) + block_idx] = 0; } } @@ -1050,6 +1049,7 @@ for (i = 0; i < TX_MODES; i++) tx_cache[i] = INT64_MAX; + vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm)); /* Y Search for intra prediction mode */ for (mode = DC_PRED; mode <= TM_PRED; mode++) { int64_t local_tx_cache[TX_MODES]; @@ -1149,6 +1149,7 @@ int this_rate_tokenonly, this_rate, s; int64_t this_distortion, this_sse; + vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm)); for (mode = DC_PRED; mode <= TM_PRED; ++mode) { if (!(cpi->sf.intra_uv_mode_mask[max_tx_size] & (1 << mode))) continue; @@ -1187,6 +1188,7 @@ int64_t unused; x->e_mbd.mi[0].src_mi->mbmi.uv_mode = DC_PRED; + vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm)); super_block_uvrd(cpi, x, rate_tokenonly, distortion, skippable, &unused, bsize, INT64_MAX); *rate = *rate_tokenonly + cpi->intra_uv_mode_cost[cm->frame_type][DC_PRED]; @@ -2649,8 +2651,6 @@ best_filter = mbmi->interp_filter; if (cm->interp_filter == SWITCHABLE && i && !intpel_mv) best_needs_copy = !best_needs_copy; - vpx_memcpy(skip_txfm, x->skip_txfm, sizeof(skip_txfm)); - vpx_memcpy(bsse, x->bsse, sizeof(bsse)); } if ((cm->interp_filter == SWITCHABLE && newbest) || @@ -2658,6 +2658,9 @@ cm->interp_filter == mbmi->interp_filter)) { pred_exists = 1; tmp_rd = best_rd; + + vpx_memcpy(skip_txfm, x->skip_txfm, sizeof(skip_txfm)); + vpx_memcpy(bsse, x->bsse, sizeof(bsse)); } } restore_dst_buf(xd, orig_dst, orig_dst_stride); @@ -3212,6 +3215,8 @@ if (ref_frame == INTRA_FRAME) { TX_SIZE uv_tx; + + vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm)); super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable, NULL, bsize, tx_cache, best_rd); @@ -3990,6 +3995,7 @@ // then dont bother looking at UV vp9_build_inter_predictors_sbuv(&x->e_mbd, mi_row, mi_col, BLOCK_8X8); + vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm)); super_block_uvrd(cpi, x, &rate_uv, &distortion_uv, &uv_skippable, &uv_sse, BLOCK_8X8, tmp_best_rdu); if (rate_uv == INT_MAX)