commit 4ff6917b938db07726354514958103824b2d3ce1
author Vignesh Venkatasubramanian <vigneshv@google.com> Thu Oct 25 15:06:06 2018 -0700
committer Vignesh Venkatasubramanian <vigneshv@google.com> Fri Oct 26 15:55:44 2018 +0000
tree 7cbcb35342ed1c45df826c69b1ea92b029f18bd0
parent f2ea6e1b2c21c3df001bb9d39e748c51e32de69a

txfm config: Fix range computation for idtx

The range computation for row transforms is wrong when column
transform happens to be identity. Use the last entry from
range_mult2_col for all transforms instead of the
max_fwd_range_mult2_col array.

This change has no effect when column transform type is not
identity.

Change-Id: I7ba09d7e13c0e04152b22656f357760abeda9c7d

av1/encoder/av1_fwd_txfm2d.c

diff --git a/av1/encoder/av1_fwd_txfm2d.c b/av1/encoder/av1_fwd_txfm2d.c
index f25a667..d282b5f 100644
--- a/av1/encoder/av1_fwd_txfm2d.c
+++ b/av1/encoder/av1_fwd_txfm2d.c
@@ -364,8 +364,6 @@
 static const int8_t fadst8_range_mult2[8] = { 0, 0, 1, 3, 3, 5, 5, 5 };
 static const int8_t fadst16_range_mult2[10] = { 0, 0, 1, 3, 3, 5, 5, 7, 7, 7 };
 
-static const int8_t max_fwd_range_mult2_col[5] = { 3, 5, 7, 9, 11 };
-
 static const int8_t fidtx4_range_mult2[1] = { 1 };
 static const int8_t fidtx8_range_mult2[1] = { 2 };
 static const int8_t fidtx16_range_mult2[1] = { 3 };
@@ -389,14 +387,12 @@
 };
 
 static INLINE void set_fwd_txfm_non_scale_range(TXFM_2D_FLIP_CFG *cfg) {
-  const int txh_idx = get_txh_idx(cfg->tx_size);
   av1_zero(cfg->stage_range_col);
   av1_zero(cfg->stage_range_row);
 
+  const int8_t *range_mult2_col = fwd_txfm_range_mult2_list[cfg->txfm_type_col];
   if (cfg->txfm_type_col != TXFM_TYPE_INVALID) {
     int stage_num_col = cfg->stage_num_col;
-    const int8_t *range_mult2_col =
-        fwd_txfm_range_mult2_list[cfg->txfm_type_col];
     for (int i = 0; i < stage_num_col; ++i)
       cfg->stage_range_col[i] = (range_mult2_col[i] + 1) >> 1;
   }
@@ -405,9 +401,11 @@
     int stage_num_row = cfg->stage_num_row;
     const int8_t *range_mult2_row =
         fwd_txfm_range_mult2_list[cfg->txfm_type_row];
-    for (int i = 0; i < stage_num_row; ++i)
+    for (int i = 0; i < stage_num_row; ++i) {
       cfg->stage_range_row[i] =
-          (max_fwd_range_mult2_col[txh_idx] + range_mult2_row[i] + 1) >> 1;
+          (range_mult2_col[cfg->stage_num_col - 1] + range_mult2_row[i] + 1) >>
+          1;
+    }
   }
 }