commit feb925fe847d76c03545dcaf9efe7acddfaca7ca
author Urvang Joshi <urvang@google.com> Mon Dec 05 10:37:29 2016 -0800
committer Urvang Joshi <urvang@google.com> Wed Jan 18 19:04:40 2017 +0000
tree b4eeb285743f6dab6c5619d80bc673be38d72c3d
parent 60a055bd5ce1364ba132305dffd6fda75eb6730f

Enable rectangular transforms for Intra also.

These are under EXT_TX + RECT_TX experiment combo.

Results
=======

Derf Set:
--------
All Intra frames: 1.8% avg improvement (and 1.78% BD-rate improvement)
Video: 0.230% avg improvement (and 0.262% BD-rate improvement)

Objective-1-fast set
--------------------
Video: 0.52 PSNR improvement

Change-Id: I1893465929858e38419f327752dc61c19b96b997

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index d18c8ca..1545aae 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1169,9 +1169,10 @@
   const TX_SIZE tx_size = is_inter ? mbmi->min_tx_size : mbmi->tx_size;
 #else
   const TX_SIZE tx_size = mbmi->tx_size;
-#endif
+#endif  // CONFIG_VAR_TX
   if (!FIXED_TX_TYPE) {
 #if CONFIG_EXT_TX
+    const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
     const BLOCK_SIZE bsize = mbmi->sb_type;
     if (get_ext_tx_types(tx_size, bsize, is_inter) > 1 && cm->base_qindex > 0 &&
         !mbmi->skip &&
@@ -1182,16 +1183,19 @@
       int eset = get_ext_tx_set(tx_size, bsize, is_inter);
       if (is_inter) {
         assert(ext_tx_used_inter[eset][mbmi->tx_type]);
-        if (eset > 0)
-          av1_write_token(
-              w, av1_ext_tx_inter_tree[eset],
-              cm->fc->inter_ext_tx_prob[eset][txsize_sqr_map[tx_size]],
-              &ext_tx_inter_encodings[eset][mbmi->tx_type]);
+        if (eset > 0) {
+          av1_write_token(w, av1_ext_tx_inter_tree[eset],
+                          cm->fc->inter_ext_tx_prob[eset][square_tx_size],
+                          &ext_tx_inter_encodings[eset][mbmi->tx_type]);
+        }
       } else if (ALLOW_INTRA_EXT_TX) {
-        if (eset > 0)
-          av1_write_token(w, av1_ext_tx_intra_tree[eset],
-                          cm->fc->intra_ext_tx_prob[eset][tx_size][mbmi->mode],
-                          &ext_tx_intra_encodings[eset][mbmi->tx_type]);
+        assert(ext_tx_used_intra[eset][mbmi->tx_type]);
+        if (eset > 0) {
+          av1_write_token(
+              w, av1_ext_tx_intra_tree[eset],
+              cm->fc->intra_ext_tx_prob[eset][square_tx_size][mbmi->mode],
+              &ext_tx_intra_encodings[eset][mbmi->tx_type]);
+        }
       }
     }
 #else
@@ -2641,6 +2645,9 @@
   unsigned int(*eob_branch_ct)[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] =
       cpi->common.counts.eob_branch[tx_size];
   int i, j, k, l, m;
+#if CONFIG_RECT_TX
+  assert(!is_rect_tx(tx_size));
+#endif  // CONFIG_RECT_TX
 
   for (i = 0; i < PLANE_TYPES; ++i) {
     for (j = 0; j < REF_TYPES; ++j) {
@@ -2679,6 +2686,9 @@
 #else
   const int probwt = 1;
 #endif
+#if CONFIG_RECT_TX
+  assert(!is_rect_tx(tx_size));
+#endif  // CONFIG_RECT_TX
 
   switch (cpi->sf.use_fast_coef_updates) {
     case TWO_LOOP: {