Fix invalid tx_type returned by get_tx_type
1) Check if tx_type is valid in get_tx_type
2) Remove scan_order from rdcost_block_args
When lv_map is on, scan_order depends on tx_type but tx_type is
not decided before entering block_rd_txfm yet. Therefore
assigning a scan_order into rdcost_block_args and then passing it
into block_rd_txfm will cause error.
3) Pass correct index into intra_mode_to_tx_type_context in
get_tx_type
This CL doesn't affect baseline/supertx's stats.
Change-Id: I59eb12aaf1edd9110ce7a92ce61f81bf89cd5920
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 997b6c7..bc88a4a 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -901,11 +901,10 @@
static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd,
int block, TX_SIZE tx_size) {
-#if !CONFIG_LV_MAP
const MODE_INFO *const mi = xd->mi[0];
const MB_MODE_INFO *const mbmi = &mi->mbmi;
-
const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block);
+#if !CONFIG_LV_MAP
if (FIXED_TX_TYPE)
return get_default_tx_type(plane_type, xd, block_raster_idx, tx_size);
@@ -956,21 +955,23 @@
#endif // CONFIG_EXT_TX
#else // !CONFIG_LV_MAP
(void)tx_size;
- const MODE_INFO *const mi = xd->mi[0];
- const MB_MODE_INFO *const mbmi = &mi->mbmi;
-
+ TX_TYPE tx_type;
if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] ||
- mbmi->tx_size >= TX_32X32)
- return DCT_DCT;
-
- if (mbmi->sb_type < BLOCK_8X8) {
- if (mbmi->ref_frame[0] < LAST_FRAME)
- return intra_mode_to_tx_type_context[mi->bmi[block].as_mode];
- else
- return DCT_DCT;
+ mbmi->tx_size >= TX_32X32) {
+ tx_type = DCT_DCT;
+ } else if (mbmi->sb_type < BLOCK_8X8) {
+ if (is_inter_block(mbmi)) // Sub8x8-Inter
+ tx_type = DCT_DCT;
+ else // Sub8x8 Intra OR UV-Intra
+ tx_type =
+ intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
+ ? get_y_mode(mi, block_raster_idx)
+ : mbmi->uv_mode];
+ } else {
+ tx_type = mbmi->txk_type[block];
}
-
- return mbmi->txk_type[block];
+ assert(tx_type >= DCT_DCT && tx_type < TX_TYPES);
+ return tx_type;
#endif // !CONFIG_LV_MAP
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index c86a4d0..f61cb55 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -139,7 +139,6 @@
int64_t best_rd;
int exit_early;
int use_fast_coef_costing;
- const SCAN_ORDER *scan_order;
};
#define LAST_NEW_MV_INDEX 6
@@ -1534,15 +1533,6 @@
}
}
-#if !CONFIG_PVQ
-static int rate_block(int plane, int block, const ENTROPY_CONTEXT *a,
- const ENTROPY_CONTEXT *l, TX_SIZE tx_size,
- struct rdcost_block_args *args) {
- return av1_cost_coeffs(&args->cpi->common, args->x, plane, block, tx_size,
- args->scan_order, a, l, args->use_fast_coef_costing);
-}
-#endif // !CONFIG_PVQ
-
static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) {
struct rdcost_block_args *args = arg;
@@ -1550,6 +1540,9 @@
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const AV1_COMP *cpi = args->cpi;
+#if !CONFIG_LV_MAP || !CONFIG_PVQ
+ const AV1_COMMON *cm = &cpi->common;
+#endif
int64_t rd1, rd2, rd;
int coeff_ctx = combine_entropy_contexts(*(args->t_above + blk_col),
*(args->t_left + blk_row));
@@ -1568,7 +1561,6 @@
#if !CONFIG_LV_MAP
// full forward transform and quantization
- const AV1_COMMON *cm = &cpi->common;
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
coeff_ctx, AV1_XFORM_QUANT_FP);
if (x->plane[plane].eobs[block] && !xd->lossless[mbmi->segment_id])
@@ -1599,7 +1591,12 @@
#if !CONFIG_PVQ
ENTROPY_CONTEXT *a = args->t_above + blk_col;
ENTROPY_CONTEXT *l = args->t_left + blk_row;
- this_rd_stats.rate = rate_block(plane, block, a, l, tx_size, args);
+ PLANE_TYPE plane_type = get_plane_type(plane);
+ TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
+ const SCAN_ORDER *scan_order =
+ get_scan(cm, tx_size, tx_type, is_inter_block(mbmi));
+ this_rd_stats.rate = av1_cost_coeffs(cm, x, plane, block, tx_size, scan_order,
+ a, l, args->use_fast_coef_costing);
#if CONFIG_RD_DEBUG
av1_update_txb_coeff_cost(&this_rd_stats, plane, tx_size, blk_row, blk_col,
this_rd_stats.rate);
@@ -1731,10 +1728,8 @@
RD_STATS *rd_stats, int64_t ref_best_rd, int plane,
BLOCK_SIZE bsize, TX_SIZE tx_size,
int use_fast_coef_casting) {
- const AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
const struct macroblockd_plane *const pd = &xd->plane[plane];
- TX_TYPE tx_type;
struct rdcost_block_args args;
av1_zero(args);
args.x = x;
@@ -1747,10 +1742,6 @@
av1_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);
- tx_type = get_tx_type(pd->plane_type, xd, 0, tx_size);
- args.scan_order =
- get_scan(cm, tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
-
#if CONFIG_DAALA_DIST
if (plane == 0 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4))
@@ -1774,12 +1765,9 @@
int64_t *sse, int64_t ref_best_rd, int plane,
BLOCK_SIZE bsize, TX_SIZE tx_size,
int use_fast_coef_casting) {
- const AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *const xd = &x->e_mbd;
const struct macroblockd_plane *const pd = &xd->plane[plane];
struct rdcost_block_args args;
- TX_TYPE tx_type;
-
av1_zero(args);
args.cpi = cpi;
args.x = x;
@@ -1794,10 +1782,6 @@
av1_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);
- tx_type = get_tx_type(pd->plane_type, xd, 0, tx_size);
- args.scan_order =
- get_scan(cm, tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
-
block_rd_txfm(plane, 0, 0, 0, get_plane_block_size(bsize, pd), tx_size,
&args);
