Make RECT_TX(>=8x8) work with VAR_TX
Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.
Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index b2cb6ff..e1593e3 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -857,6 +857,12 @@
},
};
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
+// the probability of (0) using recursive square tx partition vs.
+// (1) biggest rect tx for 4X8-8X4/8X16-16X8/16X32-32X16 blocks
+static const aom_prob default_rect_tx_prob[TX_SIZES - 1] = { 192, 192, 192 };
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
+
int av1_get_palette_color_context(const uint8_t *color_map, int cols, int r,
int c, int n, int *color_order) {
int i, j, max, max_idx, temp;
@@ -1298,6 +1304,9 @@
av1_copy(fc->tx_size_probs, default_tx_size_prob);
#if CONFIG_VAR_TX
av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ av1_copy(fc->rect_tx_prob, default_rect_tx_prob);
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
av1_copy(fc->skip_probs, default_skip_probs);
#if CONFIG_REF_MV
@@ -1451,6 +1460,15 @@
}
#endif // CONFIG_EXT_INTER
+#if CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (cm->tx_mode == TX_MODE_SELECT) {
+ for (i = 0; i < TX_SIZES - 1; ++i) {
+ fc->rect_tx_prob[i] =
+ av1_mode_mv_merge_probs(pre_fc->rect_tx_prob[i], counts->rect_tx[i]);
+ }
+ }
+#endif // CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
+
for (i = 0; i < BLOCK_SIZE_GROUPS; i++)
aom_tree_merge_probs(av1_intra_mode_tree, pre_fc->y_mode_prob[i],
counts->y_mode[i], fc->y_mode_prob[i]);
@@ -1478,10 +1496,11 @@
}
#if CONFIG_VAR_TX
- if (cm->tx_mode == TX_MODE_SELECT)
+ if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
fc->txfm_partition_prob[i] = av1_mode_mv_merge_probs(
pre_fc->txfm_partition_prob[i], counts->txfm_partition[i]);
+ }
#endif
for (i = 0; i < SKIP_CONTEXTS; ++i)
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 956bc99..7968484 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -96,6 +96,10 @@
aom_prob tx_size_probs[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES - 1];
#if CONFIG_VAR_TX
aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ // TODO(yuec) make this flag harmonize with the original syntax
+ aom_prob rect_tx_prob[TX_SIZES - 1];
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
aom_prob skip_probs[SKIP_CONTEXTS];
#if CONFIG_REF_MV
@@ -179,6 +183,9 @@
unsigned int tx_size[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
#if CONFIG_VAR_TX
unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ unsigned int rect_tx[TX_SIZES - 1][2];
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
unsigned int skip[SKIP_CONTEXTS][2];
#if CONFIG_REF_MV
diff --git a/av1/common/loopfilter.c b/av1/common/loopfilter.c
index 7086be8..2147bb8 100644
--- a/av1/common/loopfilter.c
+++ b/av1/common/loopfilter.c
@@ -1246,12 +1246,18 @@
tx_size_mask = 0;
#if CONFIG_VAR_TX
- if (is_inter_block(mbmi) && !mbmi->skip)
- tx_size =
- (plane->plane_type == PLANE_TYPE_UV)
- ? uv_txsize_lookup[sb_type][mbmi->inter_tx_size
- [blk_row][blk_col]][ss_x][ss_y]
- : mbmi->inter_tx_size[blk_row][blk_col];
+ if (is_inter_block(mbmi) && !mbmi->skip) {
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ TX_SIZE mb_tx_size = is_rect_tx(mbmi->tx_size)
+ ? mbmi->tx_size
+ : mbmi->inter_tx_size[blk_row][blk_col];
+#else
+ TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col];
+#endif
+ tx_size = (plane->plane_type == PLANE_TYPE_UV)
+ ? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y]
+ : mb_tx_size;
+ }
#if CONFIG_EXT_TX && CONFIG_RECT_TX
tx_size_r =
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 2daf4bf..8d3202c 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -1308,10 +1308,9 @@
const int step = num_4x4_blocks_txsize_lookup[max_tx_size];
int block = 0;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- const TX_SIZE tx_size =
- plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
-
- if (tx_size >= TX_SIZES) { // rect txsize is used
+ if (is_rect_tx(mbmi->tx_size)) {
+ const TX_SIZE tx_size =
+ plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
const int stepr = num_4x4_blocks_high_txsize_lookup[tx_size];
const int stepc = num_4x4_blocks_wide_txsize_lookup[tx_size];
const int max_blocks_wide =
@@ -3491,6 +3490,12 @@
#if CONFIG_VAR_TX
for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
av1_diff_update_prob(&r, &fc->txfm_partition_prob[k]);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (cm->tx_mode == TX_MODE_SELECT) {
+ for (i = 1; i < TX_SIZES - 1; ++i)
+ av1_diff_update_prob(&r, &fc->rect_tx_prob[i]);
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
for (k = 0; k < SKIP_CONTEXTS; ++k)
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 316d317..535c5b5 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -1632,13 +1632,34 @@
const int width = num_4x4_blocks_wide_lookup[bsize];
const int height = num_4x4_blocks_high_lookup[bsize];
int idx, idy;
- for (idy = 0; idy < height; idy += bs)
- for (idx = 0; idx < width; idx += bs)
- read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
- r);
+ int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
+ int use_rect_tx = 0;
+
+ if (is_rect_tx_allowed) {
+ use_rect_tx = aom_read(r, cm->fc->rect_tx_prob[tx_size_cat]);
+ if (xd->counts) {
+ ++xd->counts->rect_tx[tx_size_cat][use_rect_tx];
+ }
+ }
+
+ if (use_rect_tx) {
+ mbmi->tx_size = max_txsize_rect_lookup[bsize];
+ set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+ } else {
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+ for (idy = 0; idy < height; idy += bs)
+ for (idx = 0; idx < width; idx += bs)
+ read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
+ r);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ }
+#endif
if (xd->counts) {
const int ctx = get_tx_size_context(xd);
- ++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][mbmi->tx_size];
+ ++xd->counts
+ ->tx_size[tx_size_cat][ctx][txsize_sqr_up_map[mbmi->tx_size]];
}
} else {
if (inter_block)
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 3bd5adc..7c33aea 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1121,9 +1121,25 @@
const int width = num_4x4_blocks_wide_lookup[bsize];
const int height = num_4x4_blocks_high_lookup[bsize];
int idx, idy;
- for (idy = 0; idy < height; idy += bs)
- for (idx = 0; idx < width; idx += bs)
- write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx_allowed(mbmi)) {
+ int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+
+ aom_write(w, is_rect_tx(mbmi->tx_size),
+ cm->fc->rect_tx_prob[tx_size_cat]);
+ }
+
+ if (is_rect_tx(mbmi->tx_size)) {
+ set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+ } else {
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+ for (idy = 0; idy < height; idy += bs)
+ for (idx = 0; idx < width; idx += bs)
+ write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
write_selected_tx_size(cm, xd, w);
@@ -1650,7 +1666,7 @@
TX_SIZE tx_size =
plane ? get_uv_tx_size(mbmi, &xd->plane[plane]) : mbmi->tx_size;
- if (is_inter_block(mbmi) && tx_size < TX_SIZES) {
+ if (is_inter_block(mbmi) && !is_rect_tx(tx_size)) {
#else
if (is_inter_block(mbmi)) {
#endif
@@ -3271,6 +3287,13 @@
#if CONFIG_VAR_TX
update_txfm_partition_probs(cm, header_bc, counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (cm->tx_mode == TX_MODE_SELECT) {
+ for (i = 1; i < TX_SIZES - 1; ++i)
+ av1_cond_prob_diff_update(header_bc, &fc->rect_tx_prob[i],
+ counts->rect_tx[i]);
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
update_skip_probs(cm, header_bc, counts);
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index f172fbb..200b02c 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5031,7 +5031,7 @@
av1_encode_sb(x, AOMMAX(bsize, BLOCK_8X8));
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (mbmi->tx_size >= TX_SIZES)
+ if (is_rect_tx(mbmi->tx_size))
av1_tokenize_sb(cpi, td, t, !output_enabled, AOMMAX(bsize, BLOCK_8X8));
else
#endif
@@ -5054,8 +5054,17 @@
assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_VAR_TX
- if (is_inter)
- tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx_allowed(mbmi)) {
+ td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
+ }
+ if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_inter)
+ tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ }
+#endif
#endif
++td->counts->tx_size[tx_size_cat][tx_size_ctx][coded_tx_size];
} else {
@@ -5118,6 +5127,11 @@
is_inter_block(mbmi) && !(mbmi->skip || seg_skip)) {
if (!output_enabled)
tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx(mbmi->tx_size)) {
+ set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
TX_SIZE tx_size;
// The new intra coding scheme requires no change of transform size
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 5a21ebd..718bddb 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -791,7 +791,7 @@
#endif
#if CONFIG_VAR_TX
- // Assert not magic number (uninitialised).
+ // Assert not magic number (uninitialized).
assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);
if (x->blk_skip[plane][(blk_row << bwl) + blk_col] == 0) {
@@ -976,9 +976,6 @@
int idx, idy;
int block = 0;
int step = num_4x4_blocks_txsize_lookup[max_tx_size];
-#if CONFIG_EXT_TX && CONFIG_RECT_TX
- const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
-#endif
av1_get_entropy_contexts(bsize, TX_4X4, pd, ctx.ta[plane], ctx.tl[plane]);
#else
const struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -991,7 +988,7 @@
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- if (tx_size >= TX_SIZES) {
+ if (is_rect_tx(mbmi->tx_size)) {
av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block,
&arg);
} else {
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 0178721..ec80ee0 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2814,6 +2814,61 @@
return best_rd;
}
+// Return value 0: early termination triggered, no valid rd cost available;
+// 1: rd cost values are valid.
+static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
+ int64_t *distortion, int *skippable, int64_t *sse,
+ BLOCK_SIZE bsize, int64_t ref_best_rd) {
+ MACROBLOCKD *const xd = &x->e_mbd;
+ MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
+ const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
+ int plane;
+ int pnrate = 0, pnskip = 1;
+ int64_t pndist = 0, pnsse = 0;
+ int is_cost_valid = 1;
+
+ if (ref_best_rd < 0) is_cost_valid = 0;
+
+ if (is_inter_block(mbmi) && is_cost_valid) {
+ int plane;
+ for (plane = 1; plane < MAX_MB_PLANE; ++plane)
+ av1_subtract_plane(x, bsize, plane);
+ }
+
+ *rate = 0;
+ *distortion = 0;
+ *sse = 0;
+ *skippable = 1;
+
+ for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
+ txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
+ plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
+ if (pnrate == INT_MAX) {
+ is_cost_valid = 0;
+ break;
+ }
+ *rate += pnrate;
+ *distortion += pndist;
+ *sse += pnsse;
+ *skippable &= pnskip;
+ if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
+ RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
+ is_cost_valid = 0;
+ break;
+ }
+ }
+
+ if (!is_cost_valid) {
+ // reset cost value
+ *rate = INT_MAX;
+ *distortion = INT64_MAX;
+ *sse = INT64_MAX;
+ *skippable = 0;
+ }
+
+ return is_cost_valid;
+}
+
#if CONFIG_VAR_TX
void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
int blk_row, int blk_col, int plane, int block,
@@ -3196,6 +3251,61 @@
mbmi->tx_type = tx_type;
inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx_allowed(mbmi)) {
+ int rate_rect_tx, skippable_rect_tx = 0;
+ int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
+ int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+ TX_SIZE tx_size = max_txsize_rect_lookup[bsize];
+ TX_SIZE var_tx_size = mbmi->tx_size;
+
+ txfm_rd_in_plane(x, cpi, &rate_rect_tx, &dist_rect_tx, &skippable_rect_tx,
+ &sse_rect_tx, ref_best_rd, 0, bsize, tx_size,
+ cpi->sf.use_fast_coef_costing);
+
+ if (*rate != INT_MAX) {
+ *rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 0);
+ if (*skippable) {
+ rd = RDCOST(x->rdmult, x->rddiv, s1, *sse);
+ } else {
+ rd = RDCOST(x->rdmult, x->rddiv, *rate + s0, *dist);
+ if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+ !(*skippable))
+ rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, *sse));
+ }
+ } else {
+ rd = INT64_MAX;
+ }
+
+ if (rate_rect_tx != INT_MAX) {
+ rate_rect_tx += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 1);
+ if (skippable_rect_tx) {
+ rd_rect_tx = RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx);
+ } else {
+ rd_rect_tx =
+ RDCOST(x->rdmult, x->rddiv, rate_rect_tx + s0, dist_rect_tx);
+ if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+ !(skippable_rect_tx))
+ rd_rect_tx =
+ AOMMIN(rd_rect_tx, RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx));
+ }
+ } else {
+ rd_rect_tx = INT64_MAX;
+ }
+
+ if (rd_rect_tx < rd) {
+ *rate = rate_rect_tx;
+ *dist = dist_rect_tx;
+ *sse = sse_rect_tx;
+ *skippable = skippable_rect_tx;
+ if (!xd->lossless[mbmi->segment_id]) x->blk_skip[0][0] = *skippable;
+ mbmi->tx_size = tx_size;
+ mbmi->inter_tx_size[0][0] = mbmi->tx_size;
+ } else {
+ mbmi->tx_size = var_tx_size;
+ }
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (*rate == INT_MAX) return INT64_MAX;
@@ -3409,17 +3519,24 @@
if (ref_best_rd < 0) is_cost_valid = 0;
+ *rate = 0;
+ *distortion = 0;
+ *sse = 0;
+ *skippable = 1;
+
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+ if (is_rect_tx(mbmi->tx_size)) {
+ return super_block_uvrd(cpi, x, rate, distortion, skippable, sse, bsize,
+ ref_best_rd);
+ }
+#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+
if (is_inter_block(mbmi) && is_cost_valid) {
int plane;
for (plane = 1; plane < MAX_MB_PLANE; ++plane)
av1_subtract_plane(x, bsize, plane);
}
- *rate = 0;
- *distortion = 0;
- *sse = 0;
- *skippable = 1;
-
for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
const struct macroblockd_plane *const pd = &xd->plane[plane];
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
@@ -3477,61 +3594,6 @@
}
#endif // CONFIG_VAR_TX
-// Return value 0: early termination triggered, no valid rd cost available;
-// 1: rd cost values are valid.
-static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
- int64_t *distortion, int *skippable, int64_t *sse,
- BLOCK_SIZE bsize, int64_t ref_best_rd) {
- MACROBLOCKD *const xd = &x->e_mbd;
- MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
- const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
- int plane;
- int pnrate = 0, pnskip = 1;
- int64_t pndist = 0, pnsse = 0;
- int is_cost_valid = 1;
-
- if (ref_best_rd < 0) is_cost_valid = 0;
-
- if (is_inter_block(mbmi) && is_cost_valid) {
- int plane;
- for (plane = 1; plane < MAX_MB_PLANE; ++plane)
- av1_subtract_plane(x, bsize, plane);
- }
-
- *rate = 0;
- *distortion = 0;
- *sse = 0;
- *skippable = 1;
-
- for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
- txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
- plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
- if (pnrate == INT_MAX) {
- is_cost_valid = 0;
- break;
- }
- *rate += pnrate;
- *distortion += pndist;
- *sse += pnsse;
- *skippable &= pnskip;
- if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
- RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
- is_cost_valid = 0;
- break;
- }
- }
-
- if (!is_cost_valid) {
- // reset cost value
- *rate = INT_MAX;
- *distortion = INT64_MAX;
- *sse = INT64_MAX;
- *skippable = 0;
- }
-
- return is_cost_valid;
-}
-
static void rd_pick_palette_intra_sbuv(
AV1_COMP *cpi, MACROBLOCK *x, int dc_mode_cost,
PALETTE_MODE_INFO *palette_mode_info, uint8_t *best_palette_color_map,
