Remove txb_coeff_cost_map from RD_STATS txb_coeff_cost_map is causing stack overflow when using RD_STATS with CONFIG_RD_DEBUG on. Change-Id: Idf5766aba8ff1e491651e87ba0f8e8988a98a66b
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 710d3a3..35260bf 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -194,11 +194,6 @@ int zero_rate; #if CONFIG_RD_DEBUG int txb_coeff_cost[MAX_MB_PLANE]; - // TODO(jingning): Temporary solution to silence stack over-size warning - // in handle_inter_mode. This should be fixed after rate-distortion - // optimization refactoring. - int16_t txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE] - [TXB_COEFF_COST_MAP_SIZE]; #endif // CONFIG_RD_DEBUG } RD_STATS;
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 32af0e8..84b9633 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -381,7 +381,6 @@ #if CONFIG_RD_DEBUG TOKEN_STATS tmp_token_stats; init_token_stats(&tmp_token_stats); - token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost; token_stats->cost += tmp_token_stats.cost; #endif } else { @@ -1299,24 +1298,8 @@ static int rd_token_stats_mismatch(RD_STATS *rd_stats, TOKEN_STATS *token_stats, int plane) { if (rd_stats->txb_coeff_cost[plane] != token_stats->cost) { - int r, c; printf("\nplane %d rd_stats->txb_coeff_cost %d token_stats->cost %d\n", plane, rd_stats->txb_coeff_cost[plane], token_stats->cost); - printf("rd txb_coeff_cost_map\n"); - for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) { - for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) { - printf("%d ", rd_stats->txb_coeff_cost_map[plane][r][c]); - } - printf("\n"); - } - - printf("pack txb_coeff_cost_map\n"); - for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) { - for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) { - printf("%d ", token_stats->txb_coeff_cost_map[r][c]); - } - printf("\n"); - } return 1; } return 0;
diff --git a/av1/encoder/rd.h b/av1/encoder/rd.h index 8af31b2..bbc668b 100644 --- a/av1/encoder/rd.h +++ b/av1/encoder/rd.h
@@ -96,12 +96,6 @@ // encoded, as there will only be 1 plane for (plane = 0; plane < MAX_MB_PLANE; ++plane) { rd_stats->txb_coeff_cost[plane] = 0; - { - int r, c; - for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) - for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) - rd_stats->txb_coeff_cost_map[plane][r][c] = 0; - } } #endif } @@ -121,12 +115,6 @@ // encoded, as there will only be 1 plane for (plane = 0; plane < MAX_MB_PLANE; ++plane) { rd_stats->txb_coeff_cost[plane] = INT_MAX; - { - int r, c; - for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) - for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) - rd_stats->txb_coeff_cost_map[plane][r][c] = INT16_MAX; - } } #endif } @@ -146,18 +134,6 @@ // encoded, as there will only be 1 plane for (int plane = 0; plane < MAX_MB_PLANE; ++plane) { rd_stats_dst->txb_coeff_cost[plane] += rd_stats_src->txb_coeff_cost[plane]; - { - // TODO(angiebird): optimize this part - int r, c; - int ref_txb_coeff_cost = 0; - for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r) - for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) { - rd_stats_dst->txb_coeff_cost_map[plane][r][c] += - rd_stats_src->txb_coeff_cost_map[plane][r][c]; - ref_txb_coeff_cost += rd_stats_dst->txb_coeff_cost_map[plane][r][c]; - } - assert(ref_txb_coeff_cost == rd_stats_dst->txb_coeff_cost[plane]); - } } #endif }
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c index ef56d35..10a9a87 100644 --- a/av1/encoder/tx_search.c +++ b/av1/encoder/tx_search.c
@@ -2098,23 +2098,11 @@ #if CONFIG_RD_DEBUG static INLINE void update_txb_coeff_cost(RD_STATS *rd_stats, int plane, - TX_SIZE tx_size, int blk_row, - int blk_col, int txb_coeff_cost) { + int blk_row, int blk_col, + int txb_coeff_cost) { (void)blk_row; (void)blk_col; - (void)tx_size; rd_stats->txb_coeff_cost[plane] += txb_coeff_cost; - - { - const int txb_h = tx_size_high_unit[tx_size]; - const int txb_w = tx_size_wide_unit[tx_size]; - int idx, idy; - for (idy = 0; idy < txb_h; ++idy) - for (idx = 0; idx < txb_w; ++idx) - rd_stats->txb_coeff_cost_map[plane][blk_row + idy][blk_col + idx] = 0; - - rd_stats->txb_coeff_cost_map[plane][blk_row][blk_col] = txb_coeff_cost; - } assert(blk_row < TXB_COEFF_COST_MAP_SIZE); assert(blk_col < TXB_COEFF_COST_MAP_SIZE); } @@ -2675,7 +2663,7 @@ RDCOST(x->rdmult, zero_blk_rate, rd_stats->sse)); if (pick_skip_txfm) { #if CONFIG_RD_DEBUG - update_txb_coeff_cost(rd_stats, 0, tx_size, blk_row, blk_col, + update_txb_coeff_cost(rd_stats, 0, blk_row, blk_col, zero_blk_rate - rd_stats->rate); #endif // CONFIG_RD_DEBUG rd_stats->rate = zero_blk_rate; @@ -3175,7 +3163,7 @@ } #if CONFIG_RD_DEBUG - update_txb_coeff_cost(&this_rd_stats, plane, tx_size, blk_row, blk_col, + update_txb_coeff_cost(&this_rd_stats, plane, blk_row, blk_col, this_rd_stats.rate); #endif // CONFIG_RD_DEBUG av1_set_txb_context(x, plane, block, tx_size, a, l);