commit b8bc026c72474e8ebc5ff88dc00d08fd4c83bf76
author Debargha Mukherjee <debargha@google.com> Fri Sep 11 08:32:56 2015 -0700
committer Debargha Mukherjee <debargha@google.com> Tue Sep 15 10:00:54 2015 -0700
tree 5c55a4e74e09e461207c2aa9807c1f99c1279f49
parent 4ce81d666ebef780c47906da9ca86070c9e94874

Misc. ext_tx fixes/enhancements

derflr: +1.732% (8-bit)

Change-Id: I9c04c8249646ff96eacacfa1dcb0bd118c04e84a

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 9b27bc7..0c2f58f 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -54,7 +54,7 @@
 #define NEW_MV_DISCOUNT_FACTOR  8
 
 #if CONFIG_EXT_TX
-const double ext_tx_th = 0.99;
+const double ext_tx_th = 0.98;
 #endif
 
 typedef struct {
@@ -2736,7 +2736,7 @@
       int64_t distortion_y_tx;
       int dummy;
       int64_t best_rdcost_tx = INT64_MAX;
-      int best_ext_tx = NORM;
+      int best_ext_tx = -1;
 
       for (i = NORM; i < EXT_TX_TYPES; i++) {
         mbmi->ext_txfrm = i;
@@ -2747,7 +2747,8 @@
         rdcost_tx = RDCOST(x->rdmult, x->rddiv, rate_y_tx, distortion_y_tx);
         rdcost_tx = MIN(rdcost_tx, RDCOST(x->rdmult, x->rddiv, 0, *psse));
         assert(rdcost_tx >= 0);
-        if (rdcost_tx < best_rdcost_tx * ext_tx_th) {
+        if (rdcost_tx <
+            (best_ext_tx == NORM ? ext_tx_th : 1) * best_rdcost_tx) {
           best_ext_tx = i;
           best_rdcost_tx = rdcost_tx;
         }