Merge "Extend ext_tx experiment to intra blocks" into nextgenv2
diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 715d2e6..d01067a 100644
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -269,11 +269,16 @@
#if CONFIG_EXT_TX
if (xd->lossless || tx_size >= TX_32X32)
return DCT_DCT;
- if (is_inter_block(&mi->mbmi)) {
- return ext_tx_to_txtype[mi->mbmi.ext_txfrm];
+ if (mbmi->sb_type >= BLOCK_8X8) {
+ if (plane_type == PLANE_TYPE_Y || is_inter_block(mbmi))
+ return ext_tx_to_txtype[mbmi->ext_txfrm];
}
- return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ?
- get_y_mode(mi, block_idx) : mbmi->uv_mode];
+
+ if (is_inter_block(mbmi))
+ return DCT_DCT;
+ else
+ return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ?
+ get_y_mode(mi, block_idx) : mbmi->uv_mode];
#else
if (plane_type != PLANE_TYPE_Y || xd->lossless || is_inter_block(mbmi) ||
tx_size >= TX_32X32)
diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c
index 5bbe045..8f59fd8 100644
--- a/vp10/common/entropymode.c
+++ b/vp10/common/entropymode.c
@@ -334,7 +334,8 @@
-ALT7, -ALT8
};
-static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
+static const vpx_prob
+default_inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128 },
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
@@ -342,6 +343,76 @@
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128 },
};
+
+static const vpx_prob
+default_intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1] = {
+ {
+ { 8, 11, 24, 112, 87, 137, 127, 134,
+ 128, 86, 128, 124, 125, 133, 176, 123, },
+ { 10, 9, 39, 106, 73, 155, 163, 228,
+ 35, 62, 129, 127, 133, 114, 213, 234, },
+ { 10, 9, 14, 88, 91, 127, 151, 51,
+ 210, 89, 126, 58, 52, 116, 217, 24, },
+ { 9, 6, 29, 113, 98, 131, 149, 210,
+ 119, 60, 124, 93, 90, 143, 170, 197, },
+ { 8, 8, 38, 101, 111, 166, 167, 141,
+ 130, 105, 128, 75, 75, 118, 197, 117, },
+ { 7, 8, 39, 91, 101, 153, 166, 200,
+ 99, 77, 123, 90, 83, 144, 224, 192, },
+ { 7, 10, 26, 86, 119, 154, 130, 101,
+ 152, 91, 129, 75, 79, 137, 219, 77, },
+ { 10, 13, 20, 86, 102, 162, 112, 76,
+ 171, 86, 134, 122, 106, 124, 196, 44, },
+ { 8, 9, 33, 108, 100, 144, 148, 215,
+ 77, 60, 125, 125, 128, 126, 198, 220, },
+ { 3, 10, 29, 111, 69, 141, 204, 141,
+ 139, 93, 120, 75, 77, 163, 242, 124, },
+ },
+ {
+ { 2, 53, 18, 147, 96, 98, 136, 133,
+ 131, 120, 153, 163, 169, 137, 173, 124, },
+ { 4, 18, 34, 133, 54, 130, 179, 228,
+ 28, 72, 153, 164, 168, 118, 227, 239, },
+ { 4, 18, 13, 125, 72, 110, 176, 36,
+ 221, 104, 148, 75, 72, 117, 225, 19, },
+ { 8, 33, 24, 162, 113, 99, 147, 226,
+ 103, 85, 153, 143, 153, 124, 155, 210, },
+ { 2, 15, 35, 107, 127, 158, 192, 128,
+ 126, 116, 151, 95, 88, 182, 241, 119, },
+ { 3, 15, 36, 112, 100, 146, 194, 189,
+ 90, 98, 152, 99, 100, 165, 235, 175, },
+ { 3, 16, 29, 109, 103, 140, 182, 76,
+ 173, 104, 147, 82, 85, 159, 235, 70, },
+ { 9, 24, 14, 120, 86, 156, 161, 34,
+ 177, 121, 142, 128, 128, 126, 185, 37, },
+ { 5, 24, 29, 152, 98, 99, 174, 228,
+ 82, 76, 147, 149, 128, 132, 191, 225, },
+ { 2, 15, 29, 111, 77, 126, 200, 135,
+ 117, 93, 152, 96, 84, 191, 245, 135, },
+ },
+ {
+ { 2, 69, 13, 173, 111, 69, 137, 159,
+ 159, 146, 151, 193, 203, 131, 180, 123, },
+ { 1, 12, 33, 164, 32, 98, 204, 242,
+ 23, 99, 149, 215, 232, 110, 239, 245, },
+ { 1, 17, 9, 136, 82, 83, 171, 28,
+ 231, 128, 135, 76, 64, 118, 235, 17, },
+ { 4, 41, 17, 195, 131, 58, 161, 237,
+ 141, 97, 153, 189, 191, 117, 182, 202, },
+ { 2, 17, 36, 104, 149, 137, 217, 139,
+ 191, 119, 125, 107, 115, 223, 249, 110, },
+ { 2, 14, 24, 127, 91, 135, 219, 198,
+ 113, 91, 164, 125, 173, 211, 250, 116, },
+ { 3, 19, 24, 120, 102, 130, 209, 81,
+ 187, 95, 143, 102, 50, 190, 244, 56, },
+ { 4, 27, 10, 128, 91, 157, 181, 33,
+ 181, 150, 141, 141, 166, 114, 215, 25, },
+ { 2, 34, 27, 187, 102, 77, 210, 245,
+ 113, 107, 136, 184, 188, 121, 210, 234, },
+ { 1, 15, 22, 141, 59, 94, 208, 133,
+ 154, 95, 152, 112, 105, 191, 242, 111, },
+ },
+};
#endif // CONFIG_EXT_TX
static void init_mode_probs(FRAME_CONTEXT *fc) {
@@ -357,7 +428,8 @@
vp10_copy(fc->skip_probs, default_skip_probs);
vp10_copy(fc->inter_mode_probs, default_inter_mode_probs);
#if CONFIG_EXT_TX
- vp10_copy(fc->ext_tx_prob, default_ext_tx_prob);
+ vp10_copy(fc->inter_ext_tx_prob, default_inter_ext_tx_prob);
+ vp10_copy(fc->intra_ext_tx_prob, default_intra_ext_tx_prob);
#endif // CONFIG_EXT_TX
}
@@ -441,8 +513,13 @@
#if CONFIG_EXT_TX
for (i = TX_4X4; i <= TX_16X16; ++i) {
- vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->ext_tx_prob[i],
- counts->ext_tx[i], fc->ext_tx_prob[i]);
+ vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->inter_ext_tx_prob[i],
+ counts->inter_ext_tx[i], fc->inter_ext_tx_prob[i]);
+
+ for (j = 0; j < INTRA_MODES; ++j)
+ vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->intra_ext_tx_prob[i][j],
+ counts->intra_ext_tx[i][j],
+ fc->intra_ext_tx_prob[i][j]);
}
#endif // CONFIG_EXT_TX
}
diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h
index bdae9ee..edd9b3a 100644
--- a/vp10/common/entropymode.h
+++ b/vp10/common/entropymode.h
@@ -57,7 +57,8 @@
vpx_prob skip_probs[SKIP_CONTEXTS];
nmv_context nmvc;
#if CONFIG_EXT_TX
- vpx_prob ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1];
+ vpx_prob inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1];
+ vpx_prob intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1];
#endif // CONFIG_EXT_TX
int initialized;
} FRAME_CONTEXT;
@@ -80,7 +81,8 @@
unsigned int skip[SKIP_CONTEXTS][2];
nmv_context_counts mv;
#if CONFIG_EXT_TX
- unsigned int ext_tx[EXT_TX_SIZES][EXT_TX_TYPES];
+ unsigned int inter_ext_tx[EXT_TX_SIZES][EXT_TX_TYPES];
+ unsigned int intra_ext_tx[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES];
#endif // CONFIG_EXT_TX
} FRAME_COUNTS;
diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c
index a555129..4797d32 100644
--- a/vp10/decoder/decodeframe.c
+++ b/vp10/decoder/decodeframe.c
@@ -2070,11 +2070,18 @@
#if CONFIG_EXT_TX
static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) {
- int i, j;
+ int i, j, k;
if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
- for (j = TX_4X4; j <= TX_16X16; ++j)
- for (i = 0; i < EXT_TX_TYPES - 1; ++i)
- vp10_diff_update_prob(r, &fc->ext_tx_prob[j][i]);
+ for (i = TX_4X4; i <= TX_16X16; ++i)
+ for (j = 0; j < EXT_TX_TYPES - 1; ++j)
+ vp10_diff_update_prob(r, &fc->inter_ext_tx_prob[i][j]);
+ }
+
+ if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
+ for (i = TX_4X4; i <= TX_16X16; ++i)
+ for (j = 0; j < INTRA_MODES; ++j)
+ for (k = 0; k < EXT_TX_TYPES - 1; ++k)
+ vp10_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k]);
}
}
#endif // CONFIG_EXT_TX
diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index 4d8a1eb..beb29a0 100644
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -244,6 +244,18 @@
}
mbmi->uv_mode = read_intra_mode(r, vp10_kf_uv_mode_prob[mbmi->mode]);
+
+#if CONFIG_EXT_TX
+ if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+ mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
+ !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+ mbmi->ext_txfrm =
+ vpx_read_tree(r, vp10_ext_tx_tree,
+ cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]);
+ } else {
+ mbmi->ext_txfrm = NORM;
+ }
+#endif // CONFIG_EXT_TX
}
static int read_mv_component(vpx_reader *r,
@@ -595,28 +607,37 @@
mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r);
inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r);
mbmi->tx_size = read_tx_size(cm, xd, !mbmi->skip || !inter_block, r);
-#if CONFIG_EXT_TX
- if (inter_block &&
- mbmi->tx_size <= TX_16X16 &&
- cm->base_qindex > 0 &&
- mbmi->sb_type >= BLOCK_8X8 &&
- !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
- !mbmi->skip) {
- FRAME_COUNTS *counts = xd->counts;
- mbmi->ext_txfrm = vpx_read_tree(r,
- vp10_ext_tx_tree,
- cm->fc->ext_tx_prob[mbmi->tx_size]);
- if (counts)
- ++counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
- } else {
- mbmi->ext_txfrm = NORM;
- }
-#endif // CONFIG_EXT_TX
if (inter_block)
read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r);
else
read_intra_block_mode_info(cm, xd, mi, r);
+
+#if CONFIG_EXT_TX
+ if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+ mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
+ !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+ FRAME_COUNTS *counts = xd->counts;
+
+ if (inter_block)
+ mbmi->ext_txfrm =
+ vpx_read_tree(r, vp10_ext_tx_tree,
+ cm->fc->inter_ext_tx_prob[mbmi->tx_size]);
+ else
+ mbmi->ext_txfrm =
+ vpx_read_tree(r, vp10_ext_tx_tree,
+ cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]);
+
+ if (counts) {
+ if (inter_block)
+ ++counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
+ else
+ ++counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm];
+ }
+ } else {
+ mbmi->ext_txfrm = NORM;
+ }
+#endif // CONFIG_EXT_TX
}
void vp10_read_mode_info(VP10Decoder *const pbi, MACROBLOCKD *xd,
diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c
index 8505725..f8c274e 100644
--- a/vp10/encoder/bitstream.c
+++ b/vp10/encoder/bitstream.c
@@ -164,22 +164,39 @@
static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) {
const int savings_thresh = vp10_cost_one(GROUP_DIFF_UPDATE_PROB) -
vp10_cost_zero(GROUP_DIFF_UPDATE_PROB);
- int i;
+ int i, j;
int savings = 0;
int do_update = 0;
for (i = TX_4X4; i <= TX_16X16; ++i) {
savings += prob_diff_update_savings(
- vp10_ext_tx_tree, cm->fc->ext_tx_prob[i],
- cm->counts.ext_tx[i], EXT_TX_TYPES);
+ vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
+ cm->counts.inter_ext_tx[i], EXT_TX_TYPES);
}
do_update = savings > savings_thresh;
vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
if (do_update) {
for (i = TX_4X4; i <= TX_16X16; ++i) {
- prob_diff_update(vp10_ext_tx_tree, cm->fc->ext_tx_prob[i],
- cm->counts.ext_tx[i], EXT_TX_TYPES, w);
+ prob_diff_update(vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
+ cm->counts.inter_ext_tx[i], EXT_TX_TYPES, w);
}
}
+
+ savings = 0;
+ do_update = 0;
+
+ for (i = TX_4X4; i <= TX_16X16; ++i)
+ for (j = 0; j < INTRA_MODES; ++j)
+ savings += prob_diff_update_savings(
+ vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
+ cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES);
+ do_update = savings > savings_thresh;
+ vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
+ if (do_update) {
+ for (i = TX_4X4; i <= TX_16X16; ++i)
+ for (j = 0; j < INTRA_MODES; ++j)
+ prob_diff_update(vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
+ cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES, w);
+ }
}
#endif // CONFIG_EXT_TX
@@ -337,18 +354,6 @@
write_selected_tx_size(cm, xd, w);
}
-#if CONFIG_EXT_TX
- if (is_inter &&
- mbmi->tx_size <= TX_16X16 &&
- cm->base_qindex > 0 &&
- bsize >= BLOCK_8X8 &&
- !mbmi->skip &&
- !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
- vp10_write_token(w, vp10_ext_tx_tree, cm->fc->ext_tx_prob[mbmi->tx_size],
- &ext_tx_encodings[mbmi->ext_txfrm]);
- }
-#endif // CONFIG_EXT_TX
-
if (!is_inter) {
if (bsize >= BLOCK_8X8) {
write_intra_mode(w, mode, cm->fc->y_mode_prob[size_group_lookup[bsize]]);
@@ -412,6 +417,21 @@
}
}
}
+
+#if CONFIG_EXT_TX
+ if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+ bsize >= BLOCK_8X8 && !mbmi->skip &&
+ !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+ if (is_inter)
+ vp10_write_token(w, vp10_ext_tx_tree,
+ cm->fc->inter_ext_tx_prob[mbmi->tx_size],
+ &ext_tx_encodings[mbmi->ext_txfrm]);
+ else
+ vp10_write_token(w, vp10_ext_tx_tree,
+ cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode],
+ &ext_tx_encodings[mbmi->ext_txfrm]);
+ }
+#endif // CONFIG_EXT_TX
}
static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd,
@@ -448,6 +468,16 @@
}
write_intra_mode(w, mbmi->uv_mode, vp10_kf_uv_mode_prob[mbmi->mode]);
+
+#if CONFIG_EXT_TX
+ if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+ bsize >= BLOCK_8X8 && !mbmi->skip &&
+ !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+ vp10_write_token(w, vp10_ext_tx_tree,
+ cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode],
+ &ext_tx_encodings[mbmi->ext_txfrm]);
+ }
+#endif // CONFIG_EXT_TX
}
static void write_modes_b(VP10_COMP *cpi, const TileInfo *const tile,
diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index ceb9eb4..356baf5 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -2989,13 +2989,13 @@
++td->counts->tx.tx_totals[mbmi->tx_size];
++td->counts->tx.tx_totals[get_uv_tx_size(mbmi, &xd->plane[1])];
#if CONFIG_EXT_TX
- if (is_inter_block(mbmi) &&
- mbmi->tx_size <= TX_16X16 &&
- cm->base_qindex > 0 &&
- bsize >= BLOCK_8X8 &&
- !mbmi->skip &&
+ if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+ bsize >= BLOCK_8X8 && !mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
- ++td->counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
+ if (is_inter_block(mbmi))
+ ++td->counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
+ else
+ ++td->counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm];
}
#endif // CONFIG_EXT_TX
}
diff --git a/vp10/encoder/encoder.h b/vp10/encoder/encoder.h
index e977910..0f8076d 100644
--- a/vp10/encoder/encoder.h
+++ b/vp10/encoder/encoder.h
@@ -459,7 +459,8 @@
int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
int partition_cost[PARTITION_CONTEXTS][PARTITION_TYPES];
#if CONFIG_EXT_TX
- int ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES];
+ int inter_ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES];
+ int intra_ext_tx_costs[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES];
#endif // CONFIG_EXT_TX
int multi_arf_allowed;
diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index 82dfd23..2b549af 100644
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -85,9 +85,13 @@
vp10_cost_tokens(cpi->switchable_interp_costs[i],
fc->switchable_interp_prob[i], vp10_switchable_interp_tree);
#if CONFIG_EXT_TX
- for (i = TX_4X4; i <= TX_16X16; ++i)
- vp10_cost_tokens(cpi->ext_tx_costs[i], fc->ext_tx_prob[i],
+ for (i = TX_4X4; i <= TX_16X16; ++i) {
+ vp10_cost_tokens(cpi->inter_ext_tx_costs[i], fc->inter_ext_tx_prob[i],
vp10_ext_tx_tree);
+ for (j = 0; j < INTRA_MODES; ++j)
+ vp10_cost_tokens(cpi->intra_ext_tx_costs[i][j],
+ fc->intra_ext_tx_prob[i][j], vp10_ext_tx_tree);
+ }
#endif // CONFIG_EXT_TX
}
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 1f1405e..bec1103 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -603,6 +603,11 @@
#if CONFIG_EXT_TX
if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless) {
for (tx_type = NORM; tx_type < EXT_TX_TYPES - 1; ++tx_type) {
+ if (mbmi->ext_txfrm >= ALT11 && mbmi->ext_txfrm < ALT16 &&
+ best_tx_type == NORM) {
+ tx_type = ALT16 - 1;
+ continue;
+ }
if (tx_type >= GET_EXT_TX_TYPES(mbmi->tx_size))
continue;
@@ -613,8 +618,13 @@
if (r == INT_MAX)
continue;
- if (mbmi->tx_size <= TX_16X16)
- r += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+ if (mbmi->tx_size <= TX_16X16) {
+ if (is_inter_block(mbmi))
+ r += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+ else
+ r += cpi->intra_ext_tx_costs[mbmi->tx_size]
+ [mbmi->mode][mbmi->ext_txfrm];
+ }
if (s)
this_rd = RDCOST(x->rdmult, x->rddiv, s1, psse);
@@ -638,10 +648,14 @@
mbmi->tx_size, cpi->sf.use_fast_coef_costing);
#if CONFIG_EXT_TX
- if (is_inter_block(mbmi) && bs >= BLOCK_8X8 &&
- mbmi->tx_size <= TX_16X16 &&
- !xd->lossless && *rate != INT_MAX)
- *rate += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+ if (bs >= BLOCK_8X8 && mbmi->tx_size <= TX_16X16 &&
+ !xd->lossless && *rate != INT_MAX) {
+ if (is_inter_block(mbmi))
+ *rate += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+ else
+ *rate += cpi->intra_ext_tx_costs[mbmi->tx_size]
+ [mbmi->mode][mbmi->ext_txfrm];
+ }
#endif // CONFIG_EXT_TX
}
@@ -693,7 +707,7 @@
#if CONFIG_EXT_TX
start_tx_type = NORM;
- if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless)
+ if (bs >= BLOCK_8X8 && !xd->lossless)
end_tx_type = EXT_TX_TYPES - 1;
else
end_tx_type = NORM;
@@ -726,9 +740,12 @@
&sse, ref_best_rd, 0, bs, n,
cpi->sf.use_fast_coef_costing);
#if CONFIG_EXT_TX
- if (is_inter_block(mbmi) && bs >= BLOCK_8X8 &&
- !xd->lossless && r != INT_MAX && n < TX_32X32)
- r += cpi->ext_tx_costs[n][mbmi->ext_txfrm];
+ if (bs >= BLOCK_8X8 && !xd->lossless && r != INT_MAX && n < TX_32X32) {
+ if (is_inter_block(mbmi))
+ r += cpi->inter_ext_tx_costs[n][mbmi->ext_txfrm];
+ else
+ r += cpi->intra_ext_tx_costs[n][mbmi->mode][mbmi->ext_txfrm];
+ }
#endif // CONFIG_EXT_TX
if (r == INT_MAX)
@@ -1136,6 +1153,9 @@
int this_rate, this_rate_tokenonly, s;
int64_t this_distortion, this_rd;
TX_SIZE best_tx = TX_4X4;
+#if CONFIG_EXT_TX
+ EXT_TX_TYPE best_tx_type = NORM;
+#endif // CONFIG_EXT_TX
int *bmode_costs;
const MODE_INFO *above_mi = xd->above_mi;
const MODE_INFO *left_mi = xd->left_mi;
@@ -1161,6 +1181,9 @@
mode_selected = mode;
best_rd = this_rd;
best_tx = mic->mbmi.tx_size;
+#if CONFIG_EXT_TX
+ best_tx_type = mic->mbmi.ext_txfrm;
+#endif // CONFIG_EXT_TX
*rate = this_rate;
*rate_tokenonly = this_rate_tokenonly;
*distortion = this_distortion;
@@ -1170,6 +1193,9 @@
mic->mbmi.mode = mode_selected;
mic->mbmi.tx_size = best_tx;
+#if CONFIG_EXT_TX
+ mic->mbmi.ext_txfrm = best_tx_type;
+#endif // CONFIG_EXT_TX
return best_rd;
}
@@ -3295,9 +3321,6 @@
if (ref_frame == INTRA_FRAME) {
TX_SIZE uv_tx;
struct macroblockd_plane *const pd = &xd->plane[1];
-#if CONFIG_EXT_TX
- mbmi->ext_txfrm = NORM;
-#endif // CONFIG_EXT_TX
memset(x->skip_txfm, 0, sizeof(x->skip_txfm));
super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable,
NULL, bsize, best_rd);
@@ -3322,9 +3345,6 @@
rate2 += intra_cost_penalty;
distortion2 = distortion_y + distortion_uv;
} else {
-#if CONFIG_EXT_TX
- mbmi->ext_txfrm = NORM;
-#endif
this_rd = handle_inter_mode(cpi, x, bsize,
&rate2, &distortion2, &skippable,
&rate_y, &rate_uv,
@@ -3755,9 +3775,6 @@
int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
int internal_active_edge =
vp10_active_edge_sb(cpi, mi_row, mi_col) && vp10_internal_image_edge(cpi);
-#if CONFIG_EXT_TX
- mbmi->ext_txfrm = NORM;
-#endif
memset(x->zcoeff_blk[TX_4X4], 0, 4);
vp10_zero(best_mbmode);
@@ -3916,9 +3933,6 @@
if (ref_frame == INTRA_FRAME) {
int rate;
-#if CONFIG_EXT_TX
- mbmi->ext_txfrm = NORM;
-#endif
if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y,
&distortion_y, best_rd) >= best_rd)
continue;
