Use CDFs to cost right/bottom partitions in rd_pick_partition
This matches the code in read_partition in decodeframe.c and almost
gets rid of the last usage of the "partition_prob" tables which can
now be deleted.
Change-Id: Ibe0e947007d593d8e5f4ab10a38cd920856e566d
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 0fe5e6b..73fe6de 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -616,8 +616,6 @@
int chroma_up_available;
int chroma_left_available;
- const aom_prob (*partition_probs)[PARTITION_TYPES - 1];
-
/* Distance of MB away from frame edges in subpixels (1/8th pixel) */
int mb_to_left_edge;
int mb_to_right_edge;
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index d7a5594..dc045bc 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -690,70 +690,6 @@
#endif // CONFIG_CTX1D
#endif // CONFIG_LV_MAP
-#if CONFIG_EXT_PARTITION_TYPES
-static const aom_prob
- default_partition_probs[PARTITION_CONTEXTS][EXT_PARTITION_TYPES - 1] = {
- // 8x8 -> 4x4
- { 199, 122, 141, 128, 128, 128, 255, 128, 255 }, // a/l both not split
- { 147, 63, 159, 128, 128, 128, 255, 128, 255 }, // a split, l not split
- { 148, 133, 118, 128, 128, 128, 255, 128, 255 }, // l split, a not split
- { 121, 104, 114, 128, 128, 128, 255, 128, 255 }, // a/l both split
- // 16x16 -> 8x8
- { 174, 73, 87, 128, 128, 128, 255, 128, 255 }, // a/l both not split
- { 92, 41, 83, 128, 128, 128, 255, 128, 255 }, // a split, l not split
- { 82, 99, 50, 128, 128, 128, 255, 128, 255 }, // l split, a not split
- { 53, 39, 39, 128, 128, 128, 255, 128, 255 }, // a/l both split
- // 32x32 -> 16x16
- { 177, 58, 59, 128, 128, 85, 128, 85, 128 }, // a/l both not split
- { 68, 26, 63, 128, 128, 85, 128, 85, 128 }, // a split, l not split
- { 52, 79, 25, 128, 128, 85, 128, 85, 128 }, // l split, a not split
- { 17, 14, 12, 128, 128, 85, 128, 85, 128 }, // a/l both split
- // 64x64 -> 32x32
- { 222, 34, 30, 128, 128, 85, 128, 85, 128 }, // a/l both not split
- { 72, 16, 44, 128, 128, 85, 128, 85, 128 }, // a split, l not split
- { 58, 32, 12, 128, 128, 85, 128, 85, 128 }, // l split, a not split
- { 10, 7, 6, 128, 128, 85, 128, 85, 128 }, // a/l both split
-#if CONFIG_EXT_PARTITION
- // 128x128 -> 64x64
- { 222, 34, 30, 128, 128, 128, 255, 128, 255 }, // a/l both not split
- { 72, 16, 44, 128, 128, 128, 255, 128, 255 }, // a split, l not split
- { 58, 32, 12, 128, 128, 128, 255, 128, 255 }, // l split, a not split
- { 10, 7, 6, 128, 128, 128, 255, 128, 255 }, // a/l both split
-#endif // CONFIG_EXT_PARTITION
- };
-#else
-static const aom_prob
- default_partition_probs[PARTITION_CONTEXTS][PARTITION_TYPES - 1] = {
- // 8x8 -> 4x4
- { 199, 122, 141 }, // a/l both not split
- { 147, 63, 159 }, // a split, l not split
- { 148, 133, 118 }, // l split, a not split
- { 121, 104, 114 }, // a/l both split
- // 16x16 -> 8x8
- { 174, 73, 87 }, // a/l both not split
- { 92, 41, 83 }, // a split, l not split
- { 82, 99, 50 }, // l split, a not split
- { 53, 39, 39 }, // a/l both split
- // 32x32 -> 16x16
- { 177, 58, 59 }, // a/l both not split
- { 68, 26, 63 }, // a split, l not split
- { 52, 79, 25 }, // l split, a not split
- { 17, 14, 12 }, // a/l both split
- // 64x64 -> 32x32
- { 222, 34, 30 }, // a/l both not split
- { 72, 16, 44 }, // a split, l not split
- { 58, 32, 12 }, // l split, a not split
- { 10, 7, 6 }, // a/l both split
-#if CONFIG_EXT_PARTITION
- // 128x128 -> 64x64
- { 222, 34, 30 }, // a/l both not split
- { 72, 16, 44 }, // a split, l not split
- { 58, 32, 12 }, // l split, a not split
- { 10, 7, 6 }, // a/l both split
-#endif // CONFIG_EXT_PARTITION
- };
-#endif // CONFIG_EXT_PARTITION_TYPES
-
static const aom_prob default_newmv_prob[NEWMV_MODE_CONTEXTS] = {
155, 116, 94, 32, 96, 56, 30,
};
@@ -1029,28 +965,6 @@
};
/* clang-format on */
-const aom_tree_index av1_partition_tree[TREE_SIZE(PARTITION_TYPES)] = {
- -PARTITION_NONE, 2, -PARTITION_HORZ, 4, -PARTITION_VERT, -PARTITION_SPLIT
-};
-
-#if CONFIG_EXT_PARTITION_TYPES
-/* clang-format off */
-const aom_tree_index av1_ext_partition_tree[TREE_SIZE(EXT_PARTITION_TYPES)] = {
- -PARTITION_NONE, 2,
- 6, 4,
- 8, -PARTITION_SPLIT,
- -PARTITION_HORZ, 10,
- -PARTITION_VERT, 14,
-
- -PARTITION_HORZ_A, 12,
- -PARTITION_HORZ_B, -PARTITION_HORZ_4,
-
- -PARTITION_VERT_A, 16,
- -PARTITION_VERT_B, -PARTITION_VERT_4
-};
-/* clang-format on */
-#endif // CONFIG_EXT_PARTITION_TYPES
-
static const aom_prob default_intra_inter_p[INTRA_INTER_CONTEXTS] = {
6, 97, 151, 205,
};
@@ -3119,7 +3033,6 @@
#endif // CONFIG_EXT_INTRA_MOD
static void init_mode_probs(FRAME_CONTEXT *fc) {
- av1_copy(fc->partition_prob, default_partition_probs);
av1_copy(fc->intra_inter_prob, default_intra_inter_p);
av1_copy(fc->comp_inter_prob, default_comp_inter_p);
av1_copy(fc->palette_y_size_cdf, default_palette_y_size_cdf);
@@ -3414,19 +3327,6 @@
counts->seg.tree_total, fc->seg.tree_probs);
}
-#if CONFIG_EXT_PARTITION_TYPES
- for (i = 0; i < PARTITION_PLOFFSET; ++i)
- aom_tree_merge_probs(av1_partition_tree, pre_fc->partition_prob[i],
- counts->partition[i], fc->partition_prob[i]);
- for (; i < PARTITION_CONTEXTS_PRIMARY; ++i)
- aom_tree_merge_probs(av1_ext_partition_tree, pre_fc->partition_prob[i],
- counts->partition[i], fc->partition_prob[i]);
-#else
- for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i) {
- aom_tree_merge_probs(av1_partition_tree, pre_fc->partition_prob[i],
- counts->partition[i], fc->partition_prob[i]);
- }
-#endif // CONFIG_EXT_PARTITION_TYPES
for (i = 0; i < DELTA_Q_PROBS; ++i)
fc->delta_q_prob[i] =
mode_mv_merge_probs(pre_fc->delta_q_prob[i], counts->delta_q[i]);
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 9f5f4bf..e63a7c1 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -70,11 +70,6 @@
typedef struct frame_contexts {
aom_prob y_mode_prob[BLOCK_SIZE_GROUPS][INTRA_MODES - 1];
aom_prob uv_mode_prob[INTRA_MODES][UV_INTRA_MODES - 1];
-#if CONFIG_EXT_PARTITION_TYPES
- aom_prob partition_prob[PARTITION_CONTEXTS][EXT_PARTITION_TYPES - 1];
-#else
- aom_prob partition_prob[PARTITION_CONTEXTS][PARTITION_TYPES - 1];
-#endif
coeff_cdf_model coef_tail_cdfs[TX_SIZES][PLANE_TYPES];
coeff_cdf_model coef_head_cdfs[TX_SIZES][PLANE_TYPES];
#if CONFIG_ADAPT_SCAN
@@ -548,11 +543,6 @@
extern const aom_tree_index
av1_inter_compound_mode_tree[TREE_SIZE(INTER_COMPOUND_MODES)];
extern const aom_tree_index av1_compound_type_tree[TREE_SIZE(COMPOUND_TYPES)];
-extern const aom_tree_index av1_partition_tree[TREE_SIZE(PARTITION_TYPES)];
-#if CONFIG_EXT_PARTITION_TYPES
-extern const aom_tree_index
- av1_ext_partition_tree[TREE_SIZE(EXT_PARTITION_TYPES)];
-#endif
extern const aom_tree_index
av1_palette_color_index_tree[PALETTE_SIZES][TREE_SIZE(PALETTE_COLORS)];
extern const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES]
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index d9c6c95..2cde80a 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -2539,24 +2539,26 @@
(void)*tp_orig;
+ // Override partition costs at the edges of the frame in the same
+ // way as in read_partition (see decodeframe.c)
if (!(has_rows && has_cols)) {
- tmp_partition_cost[PARTITION_NONE] = INT_MAX;
-
+ assert(bsize_at_least_8x8 && pl >= 0);
+ const aom_cdf_prob *partition_cdf = cm->fc->partition_cdf[pl];
+ for (int i = 0; i < PARTITION_TYPES; ++i) tmp_partition_cost[i] = INT_MAX;
if (has_cols) {
- tmp_partition_cost[PARTITION_VERT] = INT_MAX;
- tmp_partition_cost[PARTITION_HORZ] =
- av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_HORZ], 0);
- tmp_partition_cost[PARTITION_SPLIT] =
- av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_HORZ], 1);
+ // At the bottom, the two possibilities are HORZ and SPLIT
+ aom_cdf_prob bot_cdf[2];
+ partition_gather_vert_alike(bot_cdf, partition_cdf);
+ static const int bot_inv_map[2] = { PARTITION_HORZ, PARTITION_SPLIT };
+ av1_cost_tokens_from_cdf(tmp_partition_cost, bot_cdf, bot_inv_map);
} else if (has_rows) {
- tmp_partition_cost[PARTITION_HORZ] = INT_MAX;
- tmp_partition_cost[PARTITION_VERT] =
- av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_VERT], 0);
- tmp_partition_cost[PARTITION_SPLIT] =
- av1_cost_bit(cm->fc->partition_prob[pl][PARTITION_VERT], 1);
+ // At the right, the two possibilities are VERT and SPLIT
+ aom_cdf_prob rhs_cdf[2];
+ partition_gather_horz_alike(rhs_cdf, partition_cdf);
+ static const int rhs_inv_map[2] = { PARTITION_VERT, PARTITION_SPLIT };
+ av1_cost_tokens_from_cdf(tmp_partition_cost, rhs_cdf, rhs_inv_map);
} else {
- tmp_partition_cost[PARTITION_HORZ] = INT_MAX;
- tmp_partition_cost[PARTITION_VERT] = INT_MAX;
+ // At the bottom right, we always split
tmp_partition_cost[PARTITION_SPLIT] = 0;
}
diff --git a/tools/aom_entropy_optimizer.c b/tools/aom_entropy_optimizer.c
index 771f3aa..8acdc0d 100644
--- a/tools/aom_entropy_optimizer.c
+++ b/tools/aom_entropy_optimizer.c
@@ -363,21 +363,12 @@
cts_each_dim[0] = PARTITION_CONTEXTS;
#if CONFIG_EXT_PARTITION_TYPES
cts_each_dim[1] = EXT_PARTITION_TYPES;
- // TODO(yuec): Wrong prob for context = 0, because the old tree is used
- optimize_entropy_table(&fc.partition[0][0], probsfile, 2, cts_each_dim,
- av1_ext_partition_tree, 0,
- "static const aom_prob default_partition_probs"
- "[PARTITION_CONTEXTS][EXT_PARTITION_TYPES - 1]");
optimize_cdf_table(&fc.partition[0][0], probsfile, 2, cts_each_dim,
"static const aom_cdf_prob\n"
"default_partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(EXT_"
"PARTITION_TYPES)]");
#else
cts_each_dim[1] = PARTITION_TYPES;
- optimize_entropy_table(&fc.partition[0][0], probsfile, 2, cts_each_dim,
- av1_partition_tree, 0,
- "static const aom_prob default_partition_probs"
- "[PARTITION_CONTEXTS][PARTITION_TYPES - 1]");
optimize_cdf_table(
&fc.partition[0][0], probsfile, 2, cts_each_dim,
"static const aom_cdf_prob\n"
