Fix invalid tx_type returned by get_tx_type
1) Check if tx_type is valid in get_tx_type
2) Remove scan_order from rdcost_block_args
When lv_map is on, scan_order depends on tx_type but tx_type is
not decided before entering block_rd_txfm yet. Therefore
assigning a scan_order into rdcost_block_args and then passing it
into block_rd_txfm will cause error.
3) Pass correct index into intra_mode_to_tx_type_context in
get_tx_type
This CL doesn't affect baseline/supertx's stats.
Change-Id: I59eb12aaf1edd9110ce7a92ce61f81bf89cd5920
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index c86a4d0..f61cb55 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -139,7 +139,6 @@
int64_t best_rd;
int exit_early;
int use_fast_coef_costing;
- const SCAN_ORDER *scan_order;
};
#define LAST_NEW_MV_INDEX 6
@@ -1534,15 +1533,6 @@
}
}
-#if !CONFIG_PVQ
-static int rate_block(int plane, int block, const ENTROPY_CONTEXT *a,
- const ENTROPY_CONTEXT *l, TX_SIZE tx_size,
- struct rdcost_block_args *args) {
- return av1_cost_coeffs(&args->cpi->common, args->x, plane, block, tx_size,
- args->scan_order, a, l, args->use_fast_coef_costing);
-}
-#endif // !CONFIG_PVQ
-
static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) {
struct rdcost_block_args *args = arg;
@@ -1550,6 +1540,9 @@
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const AV1_COMP *cpi = args->cpi;
+#if !CONFIG_LV_MAP || !CONFIG_PVQ
+ const AV1_COMMON *cm = &cpi->common;
+#endif
int64_t rd1, rd2, rd;
int coeff_ctx = combine_entropy_contexts(*(args->t_above + blk_col),
*(args->t_left + blk_row));
@@ -1568,7 +1561,6 @@
#if !CONFIG_LV_MAP
// full forward transform and quantization
- const AV1_COMMON *cm = &cpi->common;
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
coeff_ctx, AV1_XFORM_QUANT_FP);
if (x->plane[plane].eobs[block] && !xd->lossless[mbmi->segment_id])
@@ -1599,7 +1591,12 @@
#if !CONFIG_PVQ
ENTROPY_CONTEXT *a = args->t_above + blk_col;
ENTROPY_CONTEXT *l = args->t_left + blk_row;
- this_rd_stats.rate = rate_block(plane, block, a, l, tx_size, args);
+ PLANE_TYPE plane_type = get_plane_type(plane);
+ TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
+ const SCAN_ORDER *scan_order =
+ get_scan(cm, tx_size, tx_type, is_inter_block(mbmi));
+ this_rd_stats.rate = av1_cost_coeffs(cm, x, plane, block, tx_size, scan_order,
+ a, l, args->use_fast_coef_costing);
#if CONFIG_RD_DEBUG
av1_update_txb_coeff_cost(&this_rd_stats, plane, tx_size, blk_row, blk_col,
this_rd_stats.rate);
@@ -1731,10 +1728,8 @@
RD_STATS *rd_stats, int64_t ref_best_rd, int plane,
BLOCK_SIZE bsize, TX_SIZE tx_size,
int use_fast_coef_casting) {
- const AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
const struct macroblockd_plane *const pd = &xd->plane[plane];
- TX_TYPE tx_type;
struct rdcost_block_args args;
av1_zero(args);
args.x = x;
@@ -1747,10 +1742,6 @@
av1_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);
- tx_type = get_tx_type(pd->plane_type, xd, 0, tx_size);
- args.scan_order =
- get_scan(cm, tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
-
#if CONFIG_DAALA_DIST
if (plane == 0 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4))
@@ -1774,12 +1765,9 @@
int64_t *sse, int64_t ref_best_rd, int plane,
BLOCK_SIZE bsize, TX_SIZE tx_size,
int use_fast_coef_casting) {
- const AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
const struct macroblockd_plane *const pd = &xd->plane[plane];
struct rdcost_block_args args;
- TX_TYPE tx_type;
-
av1_zero(args);
args.cpi = cpi;
args.x = x;
@@ -1794,10 +1782,6 @@
av1_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);
- tx_type = get_tx_type(pd->plane_type, xd, 0, tx_size);
- args.scan_order =
- get_scan(cm, tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
-
block_rd_txfm(plane, 0, 0, 0, get_plane_block_size(bsize, pd), tx_size,
&args);
