reference_hybrid_2d: Scaling within the function.

Earlier the scaling was done outside the function. But now, we do that
within the function itself, so that the reference output can be directly
compared (without scaling) to integerized transform output.

BUG=aomedia:1114

Change-Id: I67feaf43c22c2893300336e0a9d7fa6eb2a184d9

test/av1_fwd_txfm2d_test.cc

diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc
index cf89df6..3643102 100644
--- a/test/av1_fwd_txfm2d_test.cc
+++ b/test/av1_fwd_txfm2d_test.cc
@@ -44,31 +44,14 @@
     count_ = 500;
     TXFM_2D_FLIP_CFG fwd_txfm_flip_cfg;
     av1_get_fwd_txfm_cfg(tx_type_, tx_size_, &fwd_txfm_flip_cfg);
+    amplify_factor_ = libaom_test::get_amplification_factor(tx_type_, tx_size_);
     tx_width_ = fwd_txfm_flip_cfg.row_cfg->txfm_size;
     tx_height_ = fwd_txfm_flip_cfg.col_cfg->txfm_size;
-    const int8_t *shift = (tx_width_ > tx_height_)
-                              ? fwd_txfm_flip_cfg.row_cfg->shift
-                              : fwd_txfm_flip_cfg.col_cfg->shift;
-    const int amplify_bit = shift[0] + shift[1] + shift[2];
     ud_flip_ = fwd_txfm_flip_cfg.ud_flip;
     lr_flip_ = fwd_txfm_flip_cfg.lr_flip;
-    amplify_factor_ =
-        amplify_bit >= 0 ? (1 << amplify_bit) : (1.0 / (1 << -amplify_bit));
-
-    // For rectangular transforms, we need to multiply by an extra factor.
-    const int rect_type = get_rect_tx_log_ratio(tx_width_, tx_height_);
-    if (abs(rect_type) == 1) {
-      amplify_factor_ *= pow(2, 0.5);
-    } else if (abs(rect_type) == 2) {
-      const int tx_max_dim = AOMMAX(tx_width_, tx_height_);
-      const int rect_type2_shift =
-          tx_max_dim == 64 ? 3 : tx_max_dim == 32 ? 2 : 1;
-      amplify_factor_ *= pow(2, rect_type2_shift);
-    }
 
     fwd_txfm_ = libaom_test::fwd_txfm_func_ls[tx_size_];
     txfm2d_size_ = tx_width_ * tx_height_;
-    get_txfm1d_type(tx_type_, &type0_, &type1_);
     input_ = reinterpret_cast<int16_t *>(
         aom_memalign(16, sizeof(input_[0]) * txfm2d_size_));
     output_ = reinterpret_cast<int32_t *>(
@@ -100,12 +83,12 @@
         libaom_test::flipud(ref_input_, tx_width_, tx_height_, tx_width_);
       }
 
-      reference_hybrid_2d(ref_input_, ref_output_, tx_width_, tx_height_,
-                          type0_, type1_);
+      libaom_test::reference_hybrid_2d(ref_input_, ref_output_, tx_type_,
+                                       tx_size_);
 
       double actual_max_error = 0;
       for (int ni = 0; ni < txfm2d_size_; ++ni) {
-        ref_output_[ni] = round(ref_output_[ni] * amplify_factor_);
+        ref_output_[ni] = round(ref_output_[ni]);
         const double this_error =
             fabs(output_[ni] - ref_output_[ni]) / amplify_factor_;
         actual_max_error = AOMMAX(actual_max_error, this_error);
@@ -144,8 +127,6 @@
   int tx_height_;
   int txfm2d_size_;
   Fwd_Txfm2d_Func fwd_txfm_;
-  TYPE_TXFM type0_;
-  TYPE_TXFM type1_;
   int16_t *input_;
   int32_t *output_;
   double *ref_input_;

test/av1_txfm_test.cc

diff --git a/test/av1_txfm_test.cc b/test/av1_txfm_test.cc
index 6872443..b13984b 100644
--- a/test/av1_txfm_test.cc
+++ b/test/av1_txfm_test.cc
@@ -106,8 +106,40 @@
     reference_adst_1d(in, out, size);
 }
 
-void reference_hybrid_2d(double *in, double *out, int tx_width, int tx_height,
-                         int type0, int type1) {
+double get_amplification_factor(TX_TYPE tx_type, TX_SIZE tx_size) {
+  TXFM_2D_FLIP_CFG fwd_txfm_flip_cfg;
+  av1_get_fwd_txfm_cfg(tx_type, tx_size, &fwd_txfm_flip_cfg);
+  const int tx_width = fwd_txfm_flip_cfg.row_cfg->txfm_size;
+  const int tx_height = fwd_txfm_flip_cfg.col_cfg->txfm_size;
+  const int8_t *shift = (tx_width > tx_height)
+                            ? fwd_txfm_flip_cfg.row_cfg->shift
+                            : fwd_txfm_flip_cfg.col_cfg->shift;
+  const int amplify_bit = shift[0] + shift[1] + shift[2];
+  double amplify_factor =
+      amplify_bit >= 0 ? (1 << amplify_bit) : (1.0 / (1 << -amplify_bit));
+
+  // For rectangular transforms, we need to multiply by an extra factor.
+  const int rect_type = get_rect_tx_log_ratio(tx_width, tx_height);
+  if (abs(rect_type) == 1) {
+    amplify_factor *= pow(2, 0.5);
+  } else if (abs(rect_type) == 2) {
+    const int tx_max_dim = AOMMAX(tx_width, tx_height);
+    const int rect_type2_shift =
+        tx_max_dim == 64 ? 3 : tx_max_dim == 32 ? 2 : 1;
+    amplify_factor *= pow(2, rect_type2_shift);
+  }
+  return amplify_factor;
+}
+
+void reference_hybrid_2d(double *in, double *out, TX_TYPE tx_type,
+                         TX_SIZE tx_size) {
+  // Get transform type and size of each dimension.
+  TYPE_TXFM type0;
+  TYPE_TXFM type1;
+  get_txfm1d_type(tx_type, &type0, &type1);
+  const int tx_width = tx_size_wide[tx_size];
+  const int tx_height = tx_size_high[tx_size];
+
   double *const temp_in = new double[AOMMAX(tx_width, tx_height)];
   double *const temp_out = new double[AOMMAX(tx_width, tx_height)];
   double *const out_interm = new double[tx_width * tx_height];
@@ -178,6 +210,14 @@
     }
   }
 #endif  // CONFIG_TX_64X64
+
+  // Apply appropriate scale.
+  const double amplify_factor = get_amplification_factor(tx_type, tx_size);
+  for (int c = 0; c < tx_width; ++c) {
+    for (int r = 0; r < tx_height; ++r) {
+      out[r * stride + c] *= amplify_factor;
+    }
+  }
 }
 
 template <typename Type>

test/av1_txfm_test.h

diff --git a/test/av1_txfm_test.h b/test/av1_txfm_test.h
index 527b7cf..1426f7c 100644
--- a/test/av1_txfm_test.h
+++ b/test/av1_txfm_test.h
@@ -46,8 +46,10 @@
 
 void reference_hybrid_1d(double *in, double *out, int size, int type);
 
-void reference_hybrid_2d(double *in, double *out, int tx_width, int tx_height,
-                         int type0, int type1);
+double get_amplification_factor(TX_TYPE tx_type, TX_SIZE tx_size);
+
+void reference_hybrid_2d(double *in, double *out, TX_TYPE tx_type,
+                         TX_SIZE tx_size);
 template <typename Type1, typename Type2>
 static double compute_avg_abs_error(const Type1 *a, const Type2 *b,
                                     const int size) {