Enable 1:4/4:1 transform for 8x16 and 16x8 luma blocks
It gives 0.1% gain on lowres and midres
Change-Id: I555a492a68571c525713840d73aa5614fe80a87d
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 5d723e8..4da7ba4 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -463,8 +463,6 @@
const int depth = tx_size_to_depth(coded_tx_size);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
- assert(
- IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize]));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_EC_MULTISYMBOL
@@ -475,6 +473,11 @@
ec_ctx->tx_size_probs[tx_size_cat][tx_size_ctx],
&tx_size_encodings[tx_size_cat][depth]);
#endif
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
+ aom_write(w, tx_size == quarter_txsize_lookup[bsize],
+ cm->fc->quarter_tx_size_prob);
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
}
}
@@ -4709,6 +4712,11 @@
#if !CONFIG_EC_ADAPT
update_txfm_probs(cm, header_bc, counts);
#endif
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ if (cm->tx_mode == TX_MODE_SELECT)
+ av1_cond_prob_diff_update(header_bc, &cm->fc->quarter_tx_size_prob,
+ cm->counts.quarter_tx_size, probwt);
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
#if CONFIG_LV_MAP
av1_write_txb_probs(cpi, header_bc);
#else
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index a2d5d22..8a37ca3 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5525,7 +5525,11 @@
if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0)
cm->tx_mode = ALLOW_32X32 + CONFIG_TX64X64;
#else
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ if (cm->tx_mode == TX_MODE_SELECT && counts->quarter_tx_size[1] == 0) {
+#else
if (cm->tx_mode == TX_MODE_SELECT) {
+#endif
#if CONFIG_TX64X64
int count4x4 = 0;
int count8x8_8x8p = 0, count8x8_lp = 0;
@@ -6068,6 +6072,13 @@
++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth];
#endif
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ if (is_quarter_tx_allowed(xd, mbmi, is_inter) &&
+ mbmi->tx_size != txsize_sqr_up_map[mbmi->tx_size]) {
+ ++td->counts->quarter_tx_size[mbmi->tx_size ==
+ quarter_txsize_lookup[mbmi->sb_type]];
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
#if CONFIG_EXT_TX && CONFIG_RECT_TX
assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c
index 4fd5631..2432938 100644
--- a/av1/encoder/hybrid_fwd_txfm.c
+++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -132,6 +132,36 @@
}
#endif // CONFIG_TX64X64
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+static void fwd_txfm_16x4(const int16_t *src_diff, tran_low_t *coeff,
+ int diff_stride, TX_TYPE tx_type,
+ FWD_TXFM_OPT fwd_txfm_opt) {
+ (void)fwd_txfm_opt;
+ av1_fht16x4(src_diff, coeff, diff_stride, tx_type);
+}
+
+static void fwd_txfm_4x16(const int16_t *src_diff, tran_low_t *coeff,
+ int diff_stride, TX_TYPE tx_type,
+ FWD_TXFM_OPT fwd_txfm_opt) {
+ (void)fwd_txfm_opt;
+ av1_fht4x16(src_diff, coeff, diff_stride, tx_type);
+}
+
+static void fwd_txfm_32x8(const int16_t *src_diff, tran_low_t *coeff,
+ int diff_stride, TX_TYPE tx_type,
+ FWD_TXFM_OPT fwd_txfm_opt) {
+ (void)fwd_txfm_opt;
+ av1_fht32x8(src_diff, coeff, diff_stride, tx_type);
+}
+
+static void fwd_txfm_8x32(const int16_t *src_diff, tran_low_t *coeff,
+ int diff_stride, TX_TYPE tx_type,
+ FWD_TXFM_OPT fwd_txfm_opt) {
+ (void)fwd_txfm_opt;
+ av1_fht8x32(src_diff, coeff, diff_stride, tx_type);
+}
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+
#if CONFIG_HIGHBITDEPTH
#if CONFIG_CB4X4
static void highbd_fwd_txfm_2x2(const int16_t *src_diff, tran_low_t *coeff,
@@ -430,6 +460,20 @@
fwd_txfm_2x2(src_diff, coeff, diff_stride, tx_type, lossless);
break;
#endif
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ case TX_4X16:
+ fwd_txfm_4x16(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt);
+ break;
+ case TX_16X4:
+ fwd_txfm_16x4(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt);
+ break;
+ case TX_8X32:
+ fwd_txfm_8x32(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt);
+ break;
+ case TX_32X8:
+ fwd_txfm_32x8(src_diff, coeff, diff_stride, tx_type, fwd_txfm_opt);
+ break;
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
default: assert(0); break;
}
}
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index e097392..b20562f 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -647,6 +647,34 @@
for (i = 0; i < num_4x4_h; i += 8)
t_left[i] = !!*(const uint64_t *)&left[i];
break;
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ case TX_4X16:
+ for (i = 0; i < num_4x4_w; i += 2)
+ t_above[i] = !!*(const uint16_t *)&above[i];
+ for (i = 0; i < num_4x4_h; i += 8)
+ t_left[i] = !!*(const uint64_t *)&left[i];
+ break;
+ case TX_16X4:
+ for (i = 0; i < num_4x4_w; i += 8)
+ t_above[i] = !!*(const uint64_t *)&above[i];
+ for (i = 0; i < num_4x4_h; i += 2)
+ t_left[i] = !!*(const uint16_t *)&left[i];
+ break;
+ case TX_8X32:
+ for (i = 0; i < num_4x4_w; i += 4)
+ t_above[i] = !!*(const uint32_t *)&above[i];
+ for (i = 0; i < num_4x4_h; i += 16)
+ t_left[i] =
+ !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
+ break;
+ case TX_32X8:
+ for (i = 0; i < num_4x4_w; i += 16)
+ t_above[i] =
+ !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
+ for (i = 0; i < num_4x4_h; i += 4)
+ t_left[i] = !!*(const uint32_t *)&left[i];
+ break;
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
default: assert(0 && "Invalid transform size."); break;
}
@@ -720,6 +748,30 @@
for (i = 0; i < num_4x4_h; i += 4)
t_left[i] = !!*(const uint32_t *)&left[i];
break;
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ case TX_4X16:
+ memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
+ for (i = 0; i < num_4x4_h; i += 4)
+ t_left[i] = !!*(const uint32_t *)&left[i];
+ break;
+ case TX_16X4:
+ for (i = 0; i < num_4x4_w; i += 4)
+ t_above[i] = !!*(const uint32_t *)&above[i];
+ memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
+ break;
+ case TX_8X32:
+ for (i = 0; i < num_4x4_w; i += 2)
+ t_above[i] = !!*(const uint16_t *)&above[i];
+ for (i = 0; i < num_4x4_h; i += 8)
+ t_left[i] = !!*(const uint64_t *)&left[i];
+ break;
+ case TX_32X8:
+ for (i = 0; i < num_4x4_w; i += 8)
+ t_above[i] = !!*(const uint64_t *)&above[i];
+ for (i = 0; i < num_4x4_h; i += 2)
+ t_left[i] = !!*(const uint16_t *)&left[i];
+ break;
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
default: assert(0 && "Invalid transform size."); break;
}
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 63e594a..af0a9ce 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1234,7 +1234,9 @@
BLOCK_SIZE plane_bsize, int blk_row, int blk_col,
BLOCK_SIZE tx_bsize, int *width, int *height,
int *visible_width, int *visible_height) {
+#if !(CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT)
assert(tx_bsize <= plane_bsize);
+#endif // !(CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT)
int txb_height = block_size_high[tx_bsize];
int txb_width = block_size_wide[tx_bsize];
const int block_height = block_size_high[plane_bsize];
@@ -1270,7 +1272,12 @@
&txb_cols, &txb_rows, &visible_cols, &visible_rows);
assert(visible_rows > 0);
assert(visible_cols > 0);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ if ((txb_rows == visible_rows && txb_cols == visible_cols) &&
+ tx_bsize < BLOCK_SIZES) {
+#else
if (txb_rows == visible_rows && txb_cols == visible_cols) {
+#endif
unsigned sse;
cpi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
return sse;
@@ -1795,7 +1802,12 @@
const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
const int depth = tx_size_to_depth(coded_tx_size);
const int tx_size_ctx = get_tx_size_context(xd);
- const int r_tx_size = cpi->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
+ int r_tx_size = cpi->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
+ if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
+ r_tx_size += av1_cost_bit(cm->fc->quarter_tx_size_prob,
+ tx_size == quarter_txsize_lookup[bsize]);
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
return r_tx_size;
} else {
return 0;
@@ -2198,6 +2210,56 @@
#endif // CONFIG_CB4X4 && !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
}
+
+#if CONFIG_RECT_TX_EXT
+ // test 1:4/4:1 tx
+ int evaluate_quarter_tx = 0;
+ if (is_quarter_tx_allowed(xd, mbmi, is_inter)) {
+ if (tx_select) {
+ evaluate_quarter_tx = 1;
+ } else {
+ const TX_SIZE chosen_tx_size =
+ tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
+ evaluate_quarter_tx = chosen_tx_size == quarter_txsize_lookup[bs];
+ }
+ }
+ if (evaluate_quarter_tx) {
+ TX_TYPE tx_start = DCT_DCT;
+ TX_TYPE tx_end = TX_TYPES;
+#if CONFIG_TXK_SEL
+ // The tx_type becomes dummy when lv_map is on. The tx_type search will be
+ // performed in av1_search_txk_type()
+ tx_end = DCT_DCT + 1;
+#endif
+ TX_TYPE tx_type;
+ for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
+ if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
+ const TX_SIZE tx_size = quarter_txsize_lookup[bs];
+ RD_STATS this_rd_stats;
+ int ext_tx_set =
+ get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used);
+ if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) ||
+ (!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) {
+ rd =
+ txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, tx_size);
+ if (rd < best_rd) {
+#if CONFIG_TXK_SEL
+ memcpy(best_txk_type, mbmi->txk_type,
+ sizeof(best_txk_type[0]) * num_blk);
+#endif
+ best_tx_type = tx_type;
+ best_tx_size = tx_size;
+ best_rd = rd;
+ *rd_stats = this_rd_stats;
+ }
+ }
+#if CONFIG_CB4X4 && !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
+ const int is_inter = is_inter_block(mbmi);
+ if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
+#endif // CONFIG_CB4X4 && !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
+ }
+ }
+#endif // CONFIG_RECT_TX_EXT
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (tx_select) {
@@ -3442,12 +3504,14 @@
},
};
+/* clang-format off */
static const uint8_t mode_to_angle_bin[INTRA_MODES] = {
0, 2, 6, 0, 4, 3, 5, 7, 1, 0,
#if CONFIG_ALT_INTRA
0,
#endif // CONFIG_ALT_INTRA
};
+/* clang-format on */
static void angle_estimation(const uint8_t *src, int src_stride, int rows,
int cols, BLOCK_SIZE bsize,
