commit 08542b96401a7d87bfb29592a7cd87dc7e45a9db
author Debargha Mukherjee <debargha@google.com> Tue Feb 21 01:08:14 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Wed Feb 22 17:44:35 2017 +0000
tree 27228e62ccbd1da5b9def351a3121aaa3fb95036
parent f24cea31cf8ea6785f0f399be6755d96451bf284

Reduce transform set for 16x32/32x16

Previously the transform type sets used for 16x32/32x16 transform
sizes corresponded to 16x16 transforms. Now they correspond to
32x32 transforms with a reduced available set.

BDRATE impact:
0.05% worse for lowres, 0.13% worse for midres

Change-Id: I01d405d497c13837f5404f9e0caf8c571f3b26cf

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 6506dc0..4106860 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -644,16 +644,18 @@
       if (!use_inter_ext_tx_for_txsize[s][i]) continue;
       savings += prob_diff_update_savings(
           av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
-          cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt);
+          cm->counts.inter_ext_tx[s][i],
+          num_ext_tx_set[ext_tx_set_type_inter[s]], probwt);
     }
     do_update = savings > savings_thresh;
     aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
     if (do_update) {
       for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
         if (!use_inter_ext_tx_for_txsize[s][i]) continue;
-        prob_diff_update(
-            av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
-            cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt, w);
+        prob_diff_update(av1_ext_tx_inter_tree[s],
+                         cm->fc->inter_ext_tx_prob[s][i],
+                         cm->counts.inter_ext_tx[s][i],
+                         num_ext_tx_set[ext_tx_set_type_inter[s]], probwt, w);
       }
     }
   }
@@ -666,7 +668,8 @@
       for (j = 0; j < INTRA_MODES; ++j)
         savings += prob_diff_update_savings(
             av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j],
-            cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s], probwt);
+            cm->counts.intra_ext_tx[s][i][j],
+            num_ext_tx_set[ext_tx_set_type_intra[s]], probwt);
     }
     do_update = savings > savings_thresh;
     aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
@@ -677,7 +680,7 @@
           prob_diff_update(av1_ext_tx_intra_tree[s],
                            cm->fc->intra_ext_tx_prob[s][i][j],
                            cm->counts.intra_ext_tx[s][i][j],
-                           num_ext_tx_set_intra[s], probwt, w);
+                           num_ext_tx_set[ext_tx_set_type_intra[s]], probwt, w);
       }
     }
   }