Fix enable-tx64 command line option
Fix an issue where the choose_largest_tx_size path does not take
enable_tx64 into account, which effectively disables bsize >=
BLOCK_64X64.
This issue probably still persists with chroma 444 images even after
this commit.
Change-Id: Id030a3424038f36b704e2f0db2f508f9c4bdfc66
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index ed85cf5..140ec38 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -519,6 +519,10 @@
RANGE_CHECK(extra_cfg, tx_size_search_method, 0, 2);
+ if (!extra_cfg->enable_tx64 && extra_cfg->tx_size_search_method == 2) {
+ ERROR("TX64 cannot be disabled when search_method is USE_LARGESTALL (2).");
+ }
+
for (int i = 0; i < MAX_NUM_OPERATING_POINTS; ++i) {
const int level_idx = extra_cfg->target_seq_level_idx[i];
if (!is_valid_seq_level_idx(level_idx) && level_idx != SEQ_LEVELS) {
@@ -559,6 +563,10 @@
if (img->d_w != ctx->cfg.g_w || img->d_h != ctx->cfg.g_h)
ERROR("Image size must match encoder init configuration size");
+ if (img->fmt != AOM_IMG_FMT_I420 && !ctx->extra_cfg.enable_tx64) {
+ ERROR("TX64 can only be disabled on I420 images.");
+ }
+
return AOM_CODEC_OK;
}
diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 76127fd..da80a30 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -451,7 +451,11 @@
// Used to control the tx size search evaluation for mode processing
// (normal/winner mode)
int tx_size_search_method;
- TX_MODE tx_mode;
+ // This tx_mode_search_type is used internally by the encoder, and is not
+ // written to the bitstream. It determines what kind of tx_mode should be
+ // searched. For example, we might set it to TX_MODE_LARGEST to find a good
+ // candidate, then use TX_MODE_SELECT on it
+ TX_MODE tx_mode_search_type;
// Used to control aggressiveness of skip flag prediction for mode processing
// (normal/winner mode)
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index dad696d..0b7ebcf 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -524,7 +524,7 @@
seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
mi_addr->segment_id =
map ? get_segment_id(cm, map, bsize, mi_row, mi_col) : 0;
- reset_tx_size(x, mi_addr, x->tx_mode);
+ reset_tx_size(x, mi_addr, x->tx_mode_search_type);
}
// Else for cyclic refresh mode update the segment map, set the segment id
// and then update the quantizer.
@@ -5165,8 +5165,6 @@
cm->coded_lossless = is_coded_lossless(cm, xd);
cm->all_lossless = cm->coded_lossless && !av1_superres_scaled(cm);
- cm->tx_mode = get_eval_tx_mode(cpi, DEFAULT_EVAL);
-
// Fix delta q resolution for the moment
cm->delta_q_info.delta_q_res = 0;
if (cpi->oxcf.deltaq_mode == DELTA_Q_OBJECTIVE)
@@ -5348,7 +5346,13 @@
}
// Set the transform size appropriately before bitstream creation
- cm->tx_mode = get_eval_tx_mode(cpi, WINNER_MODE_EVAL);
+ const MODE_EVAL_TYPE eval_type = cpi->sf.enable_winner_mode_for_tx_size_srch
+ ? WINNER_MODE_EVAL
+ : DEFAULT_EVAL;
+ const TX_SIZE_SEARCH_METHOD tx_search_type =
+ cpi->tx_size_search_methods[eval_type];
+ assert(cpi->oxcf.enable_tx64 || tx_search_type != USE_LARGESTALL);
+ cm->tx_mode = select_tx_mode(cpi, tx_search_type);
if (cpi->sf.tx_type_search.prune_tx_type_using_stats) {
const FRAME_UPDATE_TYPE update_type = get_frame_update_type(&cpi->gf_group);
@@ -5759,8 +5763,9 @@
if (!dry_run) {
if (av1_allow_intrabc(cm) && is_intrabc_block(mbmi)) td->intrabc_used = 1;
- if (x->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id] &&
- mbmi->sb_type > BLOCK_4X4 && !(is_inter && (mbmi->skip || seg_skip))) {
+ if (x->tx_mode_search_type == TX_MODE_SELECT &&
+ !xd->lossless[mbmi->segment_id] && mbmi->sb_type > BLOCK_4X4 &&
+ !(is_inter && (mbmi->skip || seg_skip))) {
if (is_inter) {
tx_partition_count_update(cm, x, bsize, mi_row, mi_col, td->counts,
tile_data->allow_update_cdf);
@@ -5790,7 +5795,7 @@
if (xd->lossless[mbmi->segment_id]) {
intra_tx_size = TX_4X4;
} else {
- intra_tx_size = tx_size_from_tx_mode(bsize, x->tx_mode);
+ intra_tx_size = tx_size_from_tx_mode(bsize, x->tx_mode_search_type);
}
} else {
intra_tx_size = mbmi->tx_size;
@@ -5805,9 +5810,9 @@
}
}
- if (x->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type) &&
- is_inter && !(mbmi->skip || seg_skip) &&
- !xd->lossless[mbmi->segment_id]) {
+ if (x->tx_mode_search_type == TX_MODE_SELECT &&
+ block_signals_txsize(mbmi->sb_type) && is_inter &&
+ !(mbmi->skip || seg_skip) && !xd->lossless[mbmi->segment_id]) {
if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
} else {
TX_SIZE tx_size = mbmi->tx_size;
@@ -5816,7 +5821,7 @@
if (xd->lossless[mbmi->segment_id]) {
tx_size = TX_4X4;
} else {
- tx_size = tx_size_from_tx_mode(bsize, x->tx_mode);
+ tx_size = tx_size_from_tx_mode(bsize, x->tx_mode_search_type);
}
} else {
tx_size = (bsize > BLOCK_4X4) ? tx_size : TX_4X4;
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index c2e9a4e..12b1cf2 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -502,10 +502,10 @@
unsigned int sse) {
MACROBLOCKD *const xd = &x->e_mbd;
TX_SIZE tx_size;
- if (x->tx_mode == TX_MODE_SELECT) {
+ if (x->tx_mode_search_type == TX_MODE_SELECT) {
if (sse > (var << 2))
tx_size = AOMMIN(max_txsize_lookup[bsize],
- tx_mode_to_biggest_tx_size[x->tx_mode]);
+ tx_mode_to_biggest_tx_size[x->tx_mode_search_type]);
else
tx_size = TX_8X8;
@@ -516,7 +516,7 @@
tx_size = TX_16X16;
} else {
tx_size = AOMMIN(max_txsize_lookup[bsize],
- tx_mode_to_biggest_tx_size[x->tx_mode]);
+ tx_mode_to_biggest_tx_size[x->tx_mode_search_type]);
}
if (bsize > BLOCK_32X32) tx_size = TX_16X16;
return AOMMIN(tx_size, TX_16X16);
@@ -1557,9 +1557,10 @@
#endif
init_mbmi(mi, this_mode, ref_frame, NONE_FRAME, cm);
- mi->tx_size = AOMMIN(AOMMIN(max_txsize_lookup[bsize],
- tx_mode_to_biggest_tx_size[x->tx_mode]),
- TX_16X16);
+ mi->tx_size =
+ AOMMIN(AOMMIN(max_txsize_lookup[bsize],
+ tx_mode_to_biggest_tx_size[x->tx_mode_search_type]),
+ TX_16X16);
memset(mi->inter_tx_size, mi->tx_size, sizeof(mi->inter_tx_size));
memset(xd->tx_type_map, DCT_DCT,
sizeof(xd->tx_type_map[0]) * ctx->num_4x4_blk);
@@ -1901,7 +1902,7 @@
PRED_BUFFER *const best_pred = best_pickmode.best_pred;
TX_SIZE intra_tx_size =
AOMMIN(AOMMIN(max_txsize_lookup[bsize],
- tx_mode_to_biggest_tx_size[x->tx_mode]),
+ tx_mode_to_biggest_tx_size[x->tx_mode_search_type]),
TX_16X16);
if (reuse_inter_pred && best_pred != NULL) {
diff --git a/av1/encoder/rd.h b/av1/encoder/rd.h
index 4bb7aa9..cc1484f 100644
--- a/av1/encoder/rd.h
+++ b/av1/encoder/rd.h
@@ -51,9 +51,10 @@
#define SWITCHABLE_INTERP_RATE_FACTOR 1
enum {
- // Default initialization
+ // Default initialization when we are not using winner mode framework. e.g.
+ // intrabc
DEFAULT_EVAL = 0,
- // Initialization for default mode evaluation
+ // Initialization for selecting winner mode
MODE_EVAL,
// Initialization for winner mode evaluation
WINNER_MODE_EVAL,
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 10d955f..6120387 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3821,7 +3821,8 @@
static int tx_size_cost(const MACROBLOCK *const x, BLOCK_SIZE bsize,
TX_SIZE tx_size) {
assert(bsize == x->e_mbd.mi[0]->sb_type);
- if (x->tx_mode != TX_MODE_SELECT || !block_signals_txsize(bsize)) return 0;
+ if (x->tx_mode_search_type != TX_MODE_SELECT || !block_signals_txsize(bsize))
+ return 0;
const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize);
const int depth = tx_size_to_depth(tx_size, bsize);
@@ -3840,8 +3841,8 @@
const int skip_ctx = av1_get_skip_context(xd);
int s0, s1;
const int is_inter = is_inter_block(mbmi);
- const int tx_select =
- x->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type);
+ const int tx_select = x->tx_mode_search_type == TX_MODE_SELECT &&
+ block_signals_txsize(mbmi->sb_type);
int ctx = txfm_partition_context(
xd->above_txfm_context, xd->left_txfm_context, mbmi->sb_type, tx_size);
const int r_tx_size =
@@ -3920,7 +3921,35 @@
BLOCK_SIZE bs) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = xd->mi[0];
- mbmi->tx_size = tx_size_from_tx_mode(bs, x->tx_mode);
+ mbmi->tx_size = tx_size_from_tx_mode(bs, x->tx_mode_search_type);
+
+ // If tx64 is not enabled, we need to go down to the next available size
+ if (!cpi->oxcf.enable_tx64) {
+ static const TX_SIZE tx_size_max_32[TX_SIZES_ALL] = {
+ TX_4X4, // 4x4 transform
+ TX_8X8, // 8x8 transform
+ TX_16X16, // 16x16 transform
+ TX_32X32, // 32x32 transform
+ TX_32X32, // 64x64 transform
+ TX_4X8, // 4x8 transform
+ TX_8X4, // 8x4 transform
+ TX_8X16, // 8x16 transform
+ TX_16X8, // 16x8 transform
+ TX_16X32, // 16x32 transform
+ TX_32X16, // 32x16 transform
+ TX_32X32, // 32x64 transform
+ TX_32X32, // 64x32 transform
+ TX_4X16, // 4x16 transform
+ TX_16X4, // 16x4 transform
+ TX_8X32, // 8x32 transform
+ TX_32X8, // 32x8 transform
+ TX_16X32, // 16x64 transform
+ TX_32X16, // 64x16 transform
+ };
+
+ mbmi->tx_size = tx_size_max_32[mbmi->tx_size];
+ }
+
const int skip_ctx = av1_get_skip_context(xd);
int s0, s1;
@@ -3987,7 +4016,7 @@
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = xd->mi[0];
const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bs];
- const int tx_select = x->tx_mode == TX_MODE_SELECT;
+ const int tx_select = x->tx_mode_search_type == TX_MODE_SELECT;
int start_tx;
int depth, init_depth;
@@ -3997,7 +4026,8 @@
is_inter_block(mbmi), &cpi->sf,
x->tx_size_search_method);
} else {
- const TX_SIZE chosen_tx_size = tx_size_from_tx_mode(bs, x->tx_mode);
+ const TX_SIZE chosen_tx_size =
+ tx_size_from_tx_mode(bs, x->tx_mode_search_type);
start_tx = chosen_tx_size;
init_depth = MAX_TX_DEPTH;
}
@@ -4431,7 +4461,7 @@
RD_STATS this_rd_stats;
int row, col;
int64_t temp_sse, this_rd;
- TX_SIZE tx_size = tx_size_from_tx_mode(bsize, x->tx_mode);
+ TX_SIZE tx_size = tx_size_from_tx_mode(bsize, x->tx_mode_search_type);
const int stepr = tx_size_high_unit[tx_size];
const int stepc = tx_size_wide_unit[tx_size];
const int max_blocks_wide = max_block_wide(xd, bsize, 0);
@@ -9469,7 +9499,8 @@
// cost and distortion
av1_subtract_plane(x, bsize, 0);
- if (x->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) {
+ if (x->tx_mode_search_type == TX_MODE_SELECT &&
+ !xd->lossless[mbmi->segment_id]) {
pick_tx_size_type_yrd(cpi, x, rd_stats_y, bsize, rd_thresh);
#if CONFIG_COLLECT_RD_STATS == 2
PrintPredictionUnitStats(cpi, tile_data, x, rd_stats_y, bsize);
@@ -11705,8 +11736,9 @@
// Set up tx_size related variables for skip-specific loop filtering.
search_state->best_mbmode.tx_size =
- block_signals_txsize(bsize) ? tx_size_from_tx_mode(bsize, x->tx_mode)
- : max_txsize_rect_lookup[bsize];
+ block_signals_txsize(bsize)
+ ? tx_size_from_tx_mode(bsize, x->tx_mode_search_type)
+ : max_txsize_rect_lookup[bsize];
memset(search_state->best_mbmode.inter_tx_size,
search_state->best_mbmode.tx_size,
sizeof(search_state->best_mbmode.inter_tx_size));
@@ -11788,7 +11820,8 @@
av1_build_obmc_inter_predictors_sb(cm, xd);
av1_subtract_plane(x, bsize, 0);
- if (x->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) {
+ if (x->tx_mode_search_type == TX_MODE_SELECT &&
+ !xd->lossless[mbmi->segment_id]) {
pick_tx_size_type_yrd(cpi, x, &rd_stats_y, bsize, INT64_MAX);
assert(rd_stats_y.rate != INT_MAX);
} else {
diff --git a/av1/encoder/rdopt.h b/av1/encoder/rdopt.h
index 381432f..e953c8c 100644
--- a/av1/encoder/rdopt.h
+++ b/av1/encoder/rdopt.h
@@ -211,24 +211,13 @@
static TX_MODE select_tx_mode(
const AV1_COMP *cpi, const TX_SIZE_SEARCH_METHOD tx_size_search_method) {
if (cpi->common.coded_lossless) return ONLY_4X4;
- if (tx_size_search_method == USE_LARGESTALL)
+ if (tx_size_search_method == USE_LARGESTALL) {
return TX_MODE_LARGEST;
- else if (tx_size_search_method == USE_FULL_RD ||
- tx_size_search_method == USE_FAST_RD)
+ } else {
+ assert(tx_size_search_method == USE_FULL_RD ||
+ tx_size_search_method == USE_FAST_RD);
return TX_MODE_SELECT;
- else
- return cpi->common.tx_mode;
-}
-
-static INLINE TX_MODE get_eval_tx_mode(const AV1_COMP *cpi,
- MODE_EVAL_TYPE eval_type) {
- TX_MODE tx_mode;
- if (cpi->sf.enable_winner_mode_for_tx_size_srch)
- tx_mode = select_tx_mode(cpi, cpi->tx_size_search_methods[eval_type]);
- else
- tx_mode = select_tx_mode(cpi, cpi->tx_size_search_methods[DEFAULT_EVAL]);
-
- return tx_mode;
+ }
}
static INLINE void set_tx_size_search_method(
@@ -242,7 +231,7 @@
else
x->tx_size_search_method = cpi->tx_size_search_methods[MODE_EVAL];
}
- x->tx_mode = select_tx_mode(cpi, x->tx_size_search_method);
+ x->tx_mode_search_type = select_tx_mode(cpi, x->tx_size_search_method);
}
static INLINE void set_tx_type_prune(const SPEED_FEATURES *sf, MACROBLOCK *x,