[TXMG] Add note to iadst4 range

bug=aomedia:1489

Change-Id: I5ffd9093ce40e8232c4bbb3e00de7dd56a3c75c8

av1/common/av1_inv_txfm1d.c

diff --git a/av1/common/av1_inv_txfm1d.c b/av1/common/av1_inv_txfm1d.c
index 93703d1..05fc8db 100644
--- a/av1/common/av1_inv_txfm1d.c
+++ b/av1/common/av1_inv_txfm1d.c
@@ -751,10 +751,11 @@
   s4 = range_check_value(sinpi[1] * x2, stage_range[1] + bit);
   s5 = range_check_value(sinpi[2] * x3, stage_range[1] + bit);
   s6 = range_check_value(sinpi[4] * x3, stage_range[1] + bit);
-  s7 = x0 - x2;  // no range check here. Range is checked in stage 3
 
   // stage 2
-  s7 = s7 + x3;  // no range check here. Range is checked in stage 3
+  // NOTICE: (x0 - x2) here may use one extra bit compared to the
+  // opt_range_row/col specified in av1_gen_inv_stage_range()
+  s7 = range_check_value((x0 - x2) + x3, stage_range[2]);
 
   // stage 3
   s0 = range_check_value(s0 + s3, stage_range[3] + bit);

av1/common/av1_inv_txfm2d.c

diff --git a/av1/common/av1_inv_txfm2d.c b/av1/common/av1_inv_txfm2d.c
index f301c5c..eab00bc 100644
--- a/av1/common/av1_inv_txfm2d.c
+++ b/av1/common/av1_inv_txfm2d.c
@@ -137,16 +137,28 @@
   for (int i = 0; i < cfg->stage_num_row && i < MAX_TXFM_STAGE_NUM; ++i) {
     int real_range_row = cfg->stage_range_row[i] + fwd_shift + bd + 1;
     (void)real_range_row;
-    // assert(opt_range_row >= real_range_row);
-    stage_range_row[i] = opt_range_row;
+    if (cfg->txfm_type_row == TXFM_TYPE_ADST4 && i == 1) {
+      // the adst4 may use 1 extra bit on top of opt_range_row at stage 1
+      // so opt_range_col >= real_range_col will not hold
+      stage_range_row[i] = opt_range_row;
+    } else {
+      assert(opt_range_row >= real_range_row);
+      stage_range_row[i] = opt_range_row;
+    }
   }
   // i < MAX_TXFM_STAGE_NUM will mute above array bounds warning
   for (int i = 0; i < cfg->stage_num_col && i < MAX_TXFM_STAGE_NUM; ++i) {
     int real_range_col =
         cfg->stage_range_col[i] + fwd_shift + shift[0] + bd + 1;
     (void)real_range_col;
-    // assert(opt_range_col >= real_range_col);
-    stage_range_col[i] = opt_range_col;
+    if (cfg->txfm_type_col == TXFM_TYPE_ADST4 && i == 1) {
+      // the adst4 may use 1 extra bit on top of opt_range_row at stage 1
+      // so opt_range_col >= real_range_col will not hold
+      stage_range_col[i] = opt_range_col;
+    } else {
+      assert(opt_range_col >= real_range_col);
+      stage_range_col[i] = opt_range_col;
+    }
   }
 }