commit a30b9ec632094a353a344a70eefa6d25384ca228
author Urvang Joshi <urvang@google.com> Fri Dec 15 15:03:58 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Sat Dec 16 04:13:23 2017 +0000
tree 93b5f6d4b197cb8f87c6a1bd3810f3d5e3a0c497
parent 15b0113b00b88690b5bfecdbc12d45dd9a505b7b

Correct scale factor for TX_16X64 and TX_64X16.

BUG=aomedia:1114

Change-Id: I7fbeb4c2da996801b945304e182403ec325f95bc

av1/encoder/av1_fwd_txfm2d.c

diff --git a/av1/encoder/av1_fwd_txfm2d.c b/av1/encoder/av1_fwd_txfm2d.c
index 1bf5a42..3f235d3 100644
--- a/av1/encoder/av1_fwd_txfm2d.c
+++ b/av1/encoder/av1_fwd_txfm2d.c
@@ -84,10 +84,9 @@
   int rect_type2_shift = 0;
   if (rect_type == 2 || rect_type == -2) {
     const int txfm_size_max = AOMMAX(txfm_size_col, txfm_size_row);
-    // For 64x16 / 16x64 shift 3 bits, for 32x8 / 8x32 shift 2 bits, for
-    // 16x4 / 4x16 shift by 1 bit.
-    rect_type2_shift =
-        (txfm_size_max == 64 ? 3 : (txfm_size_max == 32 ? 2 : 1));
+    // For 64x16 / 16x64 / 32x8 / 8x32 shift 2 bits, and
+    // For 16x4 / 4x16 shift by 1 bit.
+    rect_type2_shift = (txfm_size_max >= 32) ? 2 : 1;
   }
   // Take the shift from the larger dimension in the rectangular case.
   const int8_t *shift = (txfm_size_col > txfm_size_row) ? cfg->row_cfg->shift

test/av1_fwd_txfm2d_test.cc

diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc
index bc81f99..3f568e9 100644
--- a/test/av1_fwd_txfm2d_test.cc
+++ b/test/av1_fwd_txfm2d_test.cc
@@ -162,7 +162,7 @@
       param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7));
       param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7));
       param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 16, 1.6));
-      param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 16, 1.6));
+      param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 20, 2.0));
     }
 #endif  // CONFIG_TX64X64
   }

test/av1_inv_txfm2d_test.cc

diff --git a/test/av1_inv_txfm2d_test.cc b/test/av1_inv_txfm2d_test.cc
index 9877fd9..953ec68 100644
--- a/test/av1_inv_txfm2d_test.cc
+++ b/test/av1_inv_txfm2d_test.cc
@@ -167,8 +167,8 @@
       param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X64, 3, 0.2));
       param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X64, 3, 0.3));
       param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X32, 3, 0.31));
-      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.11));
-      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 2, 0.11));
+      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.16));
+      param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 2, 0.16));
     }
 #endif  // CONFIG_TX64X64
   }

test/av1_txfm_test.cc

diff --git a/test/av1_txfm_test.cc b/test/av1_txfm_test.cc
index b13984b..2e937cb 100644
--- a/test/av1_txfm_test.cc
+++ b/test/av1_txfm_test.cc
@@ -124,8 +124,7 @@
     amplify_factor *= pow(2, 0.5);
   } else if (abs(rect_type) == 2) {
     const int tx_max_dim = AOMMAX(tx_width, tx_height);
-    const int rect_type2_shift =
-        tx_max_dim == 64 ? 3 : tx_max_dim == 32 ? 2 : 1;
+    const int rect_type2_shift = (tx_max_dim >= 32) ? 2 : 1;
     amplify_factor *= pow(2, rect_type2_shift);
   }
   return amplify_factor;