Support rd model in txk sel search
Make the per transform block kernel selection process unified with
the rate distortion model used in preliminary mode search. This
makes the txk-sel model search space same as baseline.
Change-Id: I82a2d94e88a03c88154582575ced500197f8a409
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 73b71d7..26f7873 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2257,8 +2257,10 @@
MACROBLOCK *x, int *r, int64_t *d, int *s,
int64_t *sse, int64_t ref_best_rd) {
RD_STATS rd_stats;
+ x->rd_model = 1;
int64_t rd = txfm_yrd(cpi, x, &rd_stats, ref_best_rd, bs, DCT_DCT,
max_txsize_lookup[bs]);
+ x->rd_model = 0;
*r = rd_stats.rate;
*d = rd_stats.dist;
*s = rd_stats.skip;
@@ -2434,7 +2436,10 @@
for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
RD_STATS this_rd_stats;
if (skip_txfm_search(cpi, x, bs, tx_type, n, prune)) continue;
+
+ if (mbmi->ref_mv_idx > 0) x->rd_model = 1;
rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, n);
+ x->rd_model = 0;
// Early termination in transform size search.
if (cpi->sf.tx_size_search_breakout &&
