commit 33b39f011b1470a8bc23347b66c39c221793c72b
author Frederic Barbier <frederic.barbier@allegrodvt.com> Tue Nov 21 11:11:24 2017 +0100
committer Sebastien Alaiwan <sebastien.alaiwan@allegrodvt.com> Wed Nov 22 21:41:45 2017 +0000
tree 3f879cc2f53bc90928417fd0317a83ed2e55f412
parent d0179a6bc36278ba6702add232b107c602a7bbfc

[idct] Fix initialization of tx_set_type

Previous assumption on reduced_tx_set_used=0 led to many assertion
failures and prevented signalling reduced_tx_set_used equal to 1.

BUG=aomedia:1053

Change-Id: If9a9dff8d01ba3ec942e06559c153f06d34555f9

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index e30f112..9c0ceab 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1890,7 +1890,8 @@
 #endif  // CONFIG_HIGHBITDEPTH
 
 void av1_inverse_transform_block_facade(MACROBLOCKD *xd, int plane, int block,
-                                        int blk_row, int blk_col, int eob) {
+                                        int blk_row, int blk_col, int eob,
+                                        int reduced_tx_set) {
   struct macroblockd_plane *const pd = &xd->plane[plane];
   tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
 #if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
@@ -1907,7 +1908,8 @@
 #if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                               mrc_mask,
 #endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
-                              plane, tx_type, tx_size, dst, dst_stride, eob);
+                              plane, tx_type, tx_size, dst, dst_stride, eob,
+                              reduced_tx_set);
 }
 
 void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
@@ -2029,7 +2031,7 @@
                                     mrc_mask,
 #endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                                     plane, tx_type, tx_size, recon, MAX_TX_SIZE,
-                                    eob);
+                                    eob, cpi->common.reduced_tx_set_used);
 
 #if CONFIG_DIST_8X8
         if (x->using_dist_8x8 && plane == 0 && (bsw < 8 || bsh < 8)) {
@@ -2166,7 +2168,7 @@
   if (!is_inter_block(mbmi)) {
     struct macroblock_plane *const p = &x->plane[plane];
     av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col,
-                                       p->eobs[block]);
+                                       p->eobs[block], cm->reduced_tx_set_used);
     av1_dist_block(args->cpi, x, plane, plane_bsize, block, blk_row, blk_col,
                    tx_size, &this_rd_stats.dist, &this_rd_stats.sse,
                    OUTPUT_HAS_DECODED_PIXELS);
@@ -3799,7 +3801,7 @@
                               mrc_mask,
 #endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                               plane, tx_type, tx_size, rec_buffer, MAX_TX_SIZE,
-                              eob);
+                              eob, cm->reduced_tx_set_used);
   if (eob > 0) {
 #if CONFIG_DIST_8X8
     if (x->using_dist_8x8 && plane == 0 && (bw < 8 && bh < 8)) {