Support rd model in txk sel search

Make the per transform block kernel selection process unified with
the rate distortion model used in preliminary mode search. This
makes the txk-sel model search space same as baseline.

Change-Id: I82a2d94e88a03c88154582575ced500197f8a409

av1/encoder/block.h

diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index c01601b..501b21b 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -164,6 +164,8 @@
   // Save the transform RD search info.
   TX_RD_RECORD tx_rd_record;
 
+  int rd_model;
+
   // Also save RD info on the TX size search level for square TX sizes.
   TX_SIZE_RD_RECORD
   tx_size_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 946275b..c68ca6f 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -2365,7 +2365,7 @@
   MACROBLOCKD *xd = &x->e_mbd;
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   TX_TYPE txk_start = DCT_DCT;
-  TX_TYPE txk_end = TX_TYPES - 1;
+  TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1;
   TX_TYPE best_tx_type = txk_start;
   int64_t best_rd = INT64_MAX;
   uint8_t best_txb_ctx = 0;

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 73b71d7..26f7873 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2257,8 +2257,10 @@
                                    MACROBLOCK *x, int *r, int64_t *d, int *s,
                                    int64_t *sse, int64_t ref_best_rd) {
   RD_STATS rd_stats;
+  x->rd_model = 1;
   int64_t rd = txfm_yrd(cpi, x, &rd_stats, ref_best_rd, bs, DCT_DCT,
                         max_txsize_lookup[bs]);
+  x->rd_model = 0;
   *r = rd_stats.rate;
   *d = rd_stats.dist;
   *s = rd_stats.skip;
@@ -2434,7 +2436,10 @@
     for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
       RD_STATS this_rd_stats;
       if (skip_txfm_search(cpi, x, bs, tx_type, n, prune)) continue;
+
+      if (mbmi->ref_mv_idx > 0) x->rd_model = 1;
       rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, n);
+      x->rd_model = 0;
 
       // Early termination in transform size search.
       if (cpi->sf.tx_size_search_breakout &&