Turn off tx type selection for intra blocks by default

Coding gain on derflr drops to +1.83%.

Change-Id: If68c429f09422a70513d9f1e8e36e10c928e034a

vp10/common/blockd.h

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 3b94cc5..bb94397 100644
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -274,6 +274,8 @@
 };
 
 #if CONFIG_EXT_TX
+#define ALLOW_INTRA_EXT_TX 1
+
 static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = {
   1, 17, 10, 2
 };
@@ -368,7 +370,8 @@
       return DCT_DCT;
 
 #if CONFIG_EXT_TX
-    if (mbmi->sb_type >= BLOCK_8X8 && plane_type == PLANE_TYPE_Y)
+    if (mbmi->sb_type >= BLOCK_8X8 && plane_type == PLANE_TYPE_Y &&
+        ALLOW_INTRA_EXT_TX)
       return mbmi->tx_type;
 #endif  // CONFIG_EXT_TX
 
@@ -405,8 +408,10 @@
 #endif
     return DCT_DCT;
   if (mbmi->sb_type >= BLOCK_8X8) {
-    if (plane_type == PLANE_TYPE_Y)
+    if (plane_type == PLANE_TYPE_Y) {
+      if (is_inter_block(mbmi) || ALLOW_INTRA_EXT_TX)
       return mbmi->tx_type;
+    }
     if (is_inter_block(mbmi))
       // UV Inter only
       return (mbmi->tx_type == IDTX && tx_size == TX_32X32 ?

vp10/decoder/decodemv.c

diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index 34d2c73..d66ddd9 100644
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -422,7 +422,8 @@
 #if CONFIG_EXT_TX
     if (get_ext_tx_types(mbmi->tx_size, mbmi->sb_type, 0) > 1 &&
         cm->base_qindex > 0 && !mbmi->skip &&
-        !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+        !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
+        ALLOW_INTRA_EXT_TX) {
       FRAME_COUNTS *counts = xd->counts;
       int eset = get_ext_tx_set(mbmi->tx_size, mbmi->sb_type, 0);
       if (eset > 0) {
@@ -887,7 +888,7 @@
           if (counts)
             ++counts->inter_ext_tx[eset][mbmi->tx_size][mbmi->tx_type];
         }
-      } else {
+      } else if (ALLOW_INTRA_EXT_TX) {
         if (eset > 0) {
           mbmi->tx_type = vpx_read_tree(r, vp10_ext_tx_intra_tree[eset],
                                         cm->fc->intra_ext_tx_prob[eset]

vp10/encoder/bitstream.c

diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c
index 59edec2..a80e5bd 100644
--- a/vp10/encoder/bitstream.c
+++ b/vp10/encoder/bitstream.c
@@ -693,7 +693,7 @@
         vp10_write_token(w, vp10_ext_tx_inter_tree[eset],
                          cm->fc->inter_ext_tx_prob[eset][mbmi->tx_size],
                          &ext_tx_inter_encodings[eset][mbmi->tx_type]);
-    } else {
+    } else if (ALLOW_INTRA_EXT_TX) {
       if (eset > 0)
         vp10_write_token(
             w, vp10_ext_tx_intra_tree[eset],
@@ -790,7 +790,8 @@
 #if CONFIG_EXT_TX
   if (get_ext_tx_types(mbmi->tx_size, bsize, 0) > 1 &&
       cm->base_qindex > 0 && !mbmi->skip &&
-      !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+      !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
+      ALLOW_INTRA_EXT_TX) {
     int eset = get_ext_tx_set(mbmi->tx_size, bsize, 0);
     if (eset > 0)
       vp10_write_token(

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index e0912b5..7562b60 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -691,6 +691,10 @@
         if (!ext_tx_used_inter[ext_tx_set][tx_type])
           continue;
       } else {
+        if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) {
+          if (tx_type != intra_mode_to_tx_type_lookup[mbmi->mode])
+            continue;
+        }
         if (!ext_tx_used_intra[ext_tx_set][tx_type])
           continue;
       }
@@ -719,7 +723,7 @@
             r += cpi->inter_tx_type_costs[ext_tx_set]
                                          [mbmi->tx_size][mbmi->tx_type];
         } else {
-          if (ext_tx_set > 0)
+          if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
             r += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
                                          [mbmi->mode][mbmi->tx_type];
         }
@@ -754,12 +758,16 @@
   if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1 &&
       !xd->lossless[mbmi->segment_id] && *rate != INT_MAX) {
     int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter);
-    if (is_inter)
-      *rate += cpi->inter_tx_type_costs[ext_tx_set][mbmi->tx_size]
-                                       [mbmi->tx_type];
-    else
-      *rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
-                                       [mbmi->mode][mbmi->tx_type];
+    if (is_inter) {
+      if (ext_tx_set > 0)
+        *rate += cpi->inter_tx_type_costs[ext_tx_set][mbmi->tx_size]
+                                                      [mbmi->tx_type];
+    } else {
+      if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
+        *rate +=
+            cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
+                                                 [mbmi->mode][mbmi->tx_type];
+    }
   }
 #endif  // CONFIG_EXT_TX
 }
@@ -849,6 +857,10 @@
         if (!ext_tx_used_inter[ext_tx_set][tx_type])
           continue;
       } else {
+        if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) {
+          if (tx_type != intra_mode_to_tx_type_lookup[mbmi->mode])
+            continue;
+        }
         if (!ext_tx_used_intra[ext_tx_set][tx_type])
           continue;
       }
@@ -874,7 +886,7 @@
             r += cpi->inter_tx_type_costs[ext_tx_set]
                                          [mbmi->tx_size][mbmi->tx_type];
         } else {
-          if (ext_tx_set > 0)
+          if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
             r += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
                                          [mbmi->mode][mbmi->tx_type];
         }
@@ -2227,6 +2239,10 @@
       if (!ext_tx_used_inter[ext_tx_set][tx_type])
         continue;
     } else {
+      if (!ALLOW_INTRA_EXT_TX && bsize >= BLOCK_8X8) {
+        if (tx_type != intra_mode_to_tx_type_lookup[mbmi->mode])
+          continue;
+      }
       if (!ext_tx_used_intra[ext_tx_set][tx_type])
         continue;
     }
@@ -2251,7 +2267,7 @@
           this_rate += cpi->inter_tx_type_costs[ext_tx_set]
                                        [max_tx_size][mbmi->tx_type];
       } else {
-        if (ext_tx_set > 0)
+        if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
           this_rate += cpi->intra_tx_type_costs[ext_tx_set][max_tx_size]
                                        [mbmi->mode][mbmi->tx_type];
       }