commit dd8600f5f7a32a0ea7e96b8cca7c1262b11af012
author Jingning Han <jingning@google.com> Tue Jan 23 09:06:32 2018 -0800
committer Jingning Han <jingning@google.com> Wed Jan 24 00:15:33 2018 +0000
tree f04c1f8f753f20a5ac555ae83a3cf514d0eafa8f
parent 206d22f2c216979cd44981ebfdcc29760d9d9cd6

Support rd model in txk sel search

Make the per transform block kernel selection process unified with
the rate distortion model used in preliminary mode search. This
makes the txk-sel model search space same as baseline.

Change-Id: I82a2d94e88a03c88154582575ced500197f8a409

av1/encoder/block.h

diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index c01601b..501b21b 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -164,6 +164,8 @@
   // Save the transform RD search info.
   TX_RD_RECORD tx_rd_record;
 
+  int rd_model;
+
   // Also save RD info on the TX size search level for square TX sizes.
   TX_SIZE_RD_RECORD
   tx_size_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 946275b..c68ca6f 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -2365,7 +2365,7 @@
   MACROBLOCKD *xd = &x->e_mbd;
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   TX_TYPE txk_start = DCT_DCT;
-  TX_TYPE txk_end = TX_TYPES - 1;
+  TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1;
   TX_TYPE best_tx_type = txk_start;
   int64_t best_rd = INT64_MAX;
   uint8_t best_txb_ctx = 0;

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 73b71d7..26f7873 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2257,8 +2257,10 @@
                                    MACROBLOCK *x, int *r, int64_t *d, int *s,
                                    int64_t *sse, int64_t ref_best_rd) {
   RD_STATS rd_stats;
+  x->rd_model = 1;
   int64_t rd = txfm_yrd(cpi, x, &rd_stats, ref_best_rd, bs, DCT_DCT,
                         max_txsize_lookup[bs]);
+  x->rd_model = 0;
   *r = rd_stats.rate;
   *d = rd_stats.dist;
   *s = rd_stats.skip;
@@ -2434,7 +2436,10 @@
     for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
       RD_STATS this_rd_stats;
       if (skip_txfm_search(cpi, x, bs, tx_type, n, prune)) continue;
+
+      if (mbmi->ref_mv_idx > 0) x->rd_model = 1;
       rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, n);
+      x->rd_model = 0;
 
       // Early termination in transform size search.
       if (cpi->sf.tx_size_search_breakout &&